2024-11-08 09:49:52,026 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba
2024-11-08 09:49:52,044 main DEBUG Took 0.014256 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-08 09:49:52,044 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-08 09:49:52,045 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-08 09:49:52,047 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-08 09:49:52,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,071 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-08 09:49:52,084 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,089 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,090 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,092 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,094 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,095 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,096 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,098 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,099 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,100 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-08 09:49:52,103 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,103 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-08 09:49:52,107 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-08 09:49:52,109 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-08 09:49:52,112 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-08 09:49:52,113 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-08 09:49:52,115 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-08 09:49:52,115 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-08 09:49:52,126 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-08 09:49:52,130 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-08 09:49:52,132 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-08 09:49:52,133 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-08 09:49:52,134 main DEBUG createAppenders(={Console})
2024-11-08 09:49:52,135 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba initialized
2024-11-08 09:49:52,135 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba
2024-11-08 09:49:52,136 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba OK.
2024-11-08 09:49:52,136 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-08 09:49:52,137 main DEBUG OutputStream closed
2024-11-08 09:49:52,138 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-08 09:49:52,138 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-08 09:49:52,138 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@49c7b90e OK
2024-11-08 09:49:52,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-08 09:49:52,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-08 09:49:52,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-08 09:49:52,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-08 09:49:52,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-08 09:49:52,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-08 09:49:52,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-08 09:49:52,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-08 09:49:52,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-08 09:49:52,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-08 09:49:52,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-08 09:49:52,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-08 09:49:52,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-08 09:49:52,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-08 09:49:52,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-08 09:49:52,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-08 09:49:52,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-08 09:49:52,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-08 09:49:52,249 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-08 09:49:52,250 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@35432107) with optional ClassLoader: null
2024-11-08 09:49:52,250 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-08 09:49:52,251 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@35432107] started OK.
2024-11-08T09:49:52,266 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC timeout: 26 mins
2024-11-08 09:49:52,269 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-08 09:49:52,269 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-08T09:49:52,565 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106
2024-11-08T09:49:52,599 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a, deleteOnExit=true
2024-11-08T09:49:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/test.cache.data in system properties and HBase conf
2024-11-08T09:49:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.tmp.dir in system properties and HBase conf
2024-11-08T09:49:52,602 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir in system properties and HBase conf
2024-11-08T09:49:52,603 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-08T09:49:52,604 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-08T09:49:52,604 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-08T09:49:52,714 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-08T09:49:52,815 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-08T09:49:52,821 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-08T09:49:52,822 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-08T09:49:52,823 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-08T09:49:52,823 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T09:49:52,824 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-08T09:49:52,825 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-08T09:49:52,826 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T09:49:52,827 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T09:49:52,828 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-08T09:49:52,829 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/nfs.dump.dir in system properties and HBase conf
2024-11-08T09:49:52,829 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/java.io.tmpdir in system properties and HBase conf
2024-11-08T09:49:52,830 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T09:49:52,831 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-08T09:49:52,831 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-08T09:49:54,135 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-08T09:49:54,214 INFO  [Time-limited test {}] log.Log(170): Logging initialized @3021ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-08T09:49:54,283 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:49:54,347 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:49:54,370 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:49:54,370 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:49:54,372 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-08T09:49:54,383 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:49:54,387 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4395d44b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:49:54,388 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f93dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:49:54,560 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58dbf239{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/java.io.tmpdir/jetty-localhost-33309-hadoop-hdfs-3_4_1-tests_jar-_-any-8096562054615804035/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T09:49:54,568 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13e2962d{HTTP/1.1, (http/1.1)}{localhost:33309}
2024-11-08T09:49:54,569 INFO  [Time-limited test {}] server.Server(415): Started @3376ms
2024-11-08T09:49:55,142 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:49:55,153 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:49:55,156 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:49:55,157 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:49:55,157 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-08T09:49:55,158 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@431e53b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:49:55,159 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4dc262e0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:49:55,277 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65462677{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/java.io.tmpdir/jetty-localhost-41431-hadoop-hdfs-3_4_1-tests_jar-_-any-9263433790103757399/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:49:55,277 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@383014b{HTTP/1.1, (http/1.1)}{localhost:41431}
2024-11-08T09:49:55,278 INFO  [Time-limited test {}] server.Server(415): Started @4085ms
2024-11-08T09:49:55,324 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T09:49:55,433 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:49:55,440 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:49:55,441 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:49:55,441 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:49:55,442 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T09:49:55,442 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444b27d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:49:55,443 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6af5a446{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:49:55,568 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@513cab2c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/java.io.tmpdir/jetty-localhost-40661-hadoop-hdfs-3_4_1-tests_jar-_-any-2217432744215175178/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:49:55,568 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29a123ec{HTTP/1.1, (http/1.1)}{localhost:40661}
2024-11-08T09:49:55,569 INFO  [Time-limited test {}] server.Server(415): Started @4376ms
2024-11-08T09:49:55,571 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T09:49:55,619 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:49:55,624 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:49:55,628 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:49:55,628 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:49:55,630 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T09:49:55,631 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35e2f174{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:49:55,631 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@343b36c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:49:55,737 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@653e6301{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/java.io.tmpdir/jetty-localhost-40789-hadoop-hdfs-3_4_1-tests_jar-_-any-13999813998804844655/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:49:55,738 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@404caff2{HTTP/1.1, (http/1.1)}{localhost:40789}
2024-11-08T09:49:55,738 INFO  [Time-limited test {}] server.Server(415): Started @4546ms
2024-11-08T09:49:55,740 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T09:49:57,303 WARN  [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data3/current/BP-341519641-172.17.0.2-1731059393563/current, will proceed with Du for space computation calculation, 
2024-11-08T09:49:57,303 WARN  [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data4/current/BP-341519641-172.17.0.2-1731059393563/current, will proceed with Du for space computation calculation, 
2024-11-08T09:49:57,304 WARN  [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data1/current/BP-341519641-172.17.0.2-1731059393563/current, will proceed with Du for space computation calculation, 
2024-11-08T09:49:57,304 WARN  [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data2/current/BP-341519641-172.17.0.2-1731059393563/current, will proceed with Du for space computation calculation, 
2024-11-08T09:49:57,331 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T09:49:57,331 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T09:49:57,377 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82bdfa3d44c2691c with lease ID 0x487a3da35b11c503: Processing first storage report for DS-63bbb729-22ec-4c60-97ac-5b98aec877a0 from datanode DatanodeRegistration(127.0.0.1:41317, datanodeUuid=c8a235e3-00ab-417c-a743-4d2342d5a572, infoPort=44289, infoSecurePort=0, ipcPort=37075, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563)
2024-11-08T09:49:57,378 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82bdfa3d44c2691c with lease ID 0x487a3da35b11c503: from storage DS-63bbb729-22ec-4c60-97ac-5b98aec877a0 node DatanodeRegistration(127.0.0.1:41317, datanodeUuid=c8a235e3-00ab-417c-a743-4d2342d5a572, infoPort=44289, infoSecurePort=0, ipcPort=37075, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-08T09:49:57,379 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41de28ffbe2c1fc with lease ID 0x487a3da35b11c502: Processing first storage report for DS-f6ad176b-5513-4e2d-9f11-707d03b5efe8 from datanode DatanodeRegistration(127.0.0.1:44789, datanodeUuid=158f7428-aeb0-4ee5-93e8-ca384654f533, infoPort=45887, infoSecurePort=0, ipcPort=46105, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563)
2024-11-08T09:49:57,379 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41de28ffbe2c1fc with lease ID 0x487a3da35b11c502: from storage DS-f6ad176b-5513-4e2d-9f11-707d03b5efe8 node DatanodeRegistration(127.0.0.1:44789, datanodeUuid=158f7428-aeb0-4ee5-93e8-ca384654f533, infoPort=45887, infoSecurePort=0, ipcPort=46105, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:49:57,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x82bdfa3d44c2691c with lease ID 0x487a3da35b11c503: Processing first storage report for DS-40317cb8-c1f5-4bf6-b649-5c39aaaee56c from datanode DatanodeRegistration(127.0.0.1:41317, datanodeUuid=c8a235e3-00ab-417c-a743-4d2342d5a572, infoPort=44289, infoSecurePort=0, ipcPort=37075, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563)
2024-11-08T09:49:57,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x82bdfa3d44c2691c with lease ID 0x487a3da35b11c503: from storage DS-40317cb8-c1f5-4bf6-b649-5c39aaaee56c node DatanodeRegistration(127.0.0.1:41317, datanodeUuid=c8a235e3-00ab-417c-a743-4d2342d5a572, infoPort=44289, infoSecurePort=0, ipcPort=37075, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-08T09:49:57,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41de28ffbe2c1fc with lease ID 0x487a3da35b11c502: Processing first storage report for DS-71b42f6d-32f4-47bb-9f56-ed0c08a74d3b from datanode DatanodeRegistration(127.0.0.1:44789, datanodeUuid=158f7428-aeb0-4ee5-93e8-ca384654f533, infoPort=45887, infoSecurePort=0, ipcPort=46105, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563)
2024-11-08T09:49:57,381 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41de28ffbe2c1fc with lease ID 0x487a3da35b11c502: from storage DS-71b42f6d-32f4-47bb-9f56-ed0c08a74d3b node DatanodeRegistration(127.0.0.1:44789, datanodeUuid=158f7428-aeb0-4ee5-93e8-ca384654f533, infoPort=45887, infoSecurePort=0, ipcPort=46105, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:49:57,579 WARN  [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data5/current/BP-341519641-172.17.0.2-1731059393563/current, will proceed with Du for space computation calculation, 
2024-11-08T09:49:57,579 WARN  [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data6/current/BP-341519641-172.17.0.2-1731059393563/current, will proceed with Du for space computation calculation, 
2024-11-08T09:49:57,600 WARN  [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T09:49:57,606 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61701c46c12e18ba with lease ID 0x487a3da35b11c504: Processing first storage report for DS-007ceb83-e67c-4899-a70c-de5685f4ee62 from datanode DatanodeRegistration(127.0.0.1:35759, datanodeUuid=12abefdb-119b-4b5d-9c26-679a1cd1b111, infoPort=39631, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563)
2024-11-08T09:49:57,606 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61701c46c12e18ba with lease ID 0x487a3da35b11c504: from storage DS-007ceb83-e67c-4899-a70c-de5685f4ee62 node DatanodeRegistration(127.0.0.1:35759, datanodeUuid=12abefdb-119b-4b5d-9c26-679a1cd1b111, infoPort=39631, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-08T09:49:57,606 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61701c46c12e18ba with lease ID 0x487a3da35b11c504: Processing first storage report for DS-00e4accb-a70f-42cf-849d-4591ace76392 from datanode DatanodeRegistration(127.0.0.1:35759, datanodeUuid=12abefdb-119b-4b5d-9c26-679a1cd1b111, infoPort=39631, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563)
2024-11-08T09:49:57,607 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61701c46c12e18ba with lease ID 0x487a3da35b11c504: from storage DS-00e4accb-a70f-42cf-849d-4591ace76392 node DatanodeRegistration(127.0.0.1:35759, datanodeUuid=12abefdb-119b-4b5d-9c26-679a1cd1b111, infoPort=39631, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=1853324842;c=1731059393563), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:49:57,679 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106
2024-11-08T09:49:57,772 WARN  [Time-limited test {}] erasurecode.ErasureCodeNative(55): ISA-L support is not available in your platform... using builtin-java codec where applicable
2024-11-08T09:49:57,882 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestHBaseWalOnEC#testReadWrite[0] Thread=159, OpenFileDescriptor=393, MaxFileDescriptor=1048576, SystemLoadAverage=225, ProcessCount=11, AvailableMemoryMB=5776
2024-11-08T09:49:57,885 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-08T09:49:57,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(821): NOT STARTING DFS
2024-11-08T09:49:58,002 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/zookeeper_0, clientPort=52803, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-08T09:49:58,014 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52803
2024-11-08T09:49:58,023 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:58,026 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:58,139 WARN  [Time-limited test {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'.
2024-11-08T09:49:58,140 WARN  [Time-limited test {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'.
2024-11-08T09:49:58,184 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_468708289_22 at /127.0.0.1:53450 [Receiving block BP-341519641-172.17.0.2-1731059393563:blk_-9223372036854775792_1001] {}] datanode.DataXceiver(331): 127.0.0.1:35759:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53450 dst: /127.0.0.1:35759
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:49:58,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35759 is added to blk_-9223372036854775792_1002 (size=7)
2024-11-08T09:49:58,610 WARN  [Time-limited test {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data.
2024-11-08T09:49:58,619 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280 with version=8
2024-11-08T09:49:58,619 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/hbase-staging
2024-11-08T09:49:58,750 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-08T09:49:59,056 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:49:59,067 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,067 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,073 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:49:59,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:49:59,241 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-08T09:49:59,298 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-08T09:49:59,311 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-08T09:49:59,316 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:49:59,343 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 93298 (auto-detected)
2024-11-08T09:49:59,344 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-08T09:49:59,362 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43985
2024-11-08T09:49:59,382 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43985 connecting to ZooKeeper ensemble=127.0.0.1:52803
2024-11-08T09:49:59,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:439850x0, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:49:59,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43985-0x10119da31030000 connected
2024-11-08T09:49:59,585 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,587 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:49:59,602 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280, hbase.cluster.distributed=false
2024-11-08T09:49:59,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:49:59,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43985
2024-11-08T09:49:59,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43985
2024-11-08T09:49:59,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43985
2024-11-08T09:49:59,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43985
2024-11-08T09:49:59,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43985
2024-11-08T09:49:59,745 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:49:59,747 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,747 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,748 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:49:59,748 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,748 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:49:59,751 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T09:49:59,754 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:49:59,755 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42175
2024-11-08T09:49:59,757 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42175 connecting to ZooKeeper ensemble=127.0.0.1:52803
2024-11-08T09:49:59,758 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,764 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421750x0, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:49:59,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421750x0, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:49:59,785 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42175-0x10119da31030001 connected
2024-11-08T09:49:59,790 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T09:49:59,801 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T09:49:59,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:49:59,816 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:49:59,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42175
2024-11-08T09:49:59,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42175
2024-11-08T09:49:59,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42175
2024-11-08T09:49:59,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42175
2024-11-08T09:49:59,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42175
2024-11-08T09:49:59,838 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:49:59,839 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,839 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,839 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:49:59,840 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,840 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:49:59,840 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T09:49:59,840 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:49:59,841 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38441
2024-11-08T09:49:59,844 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38441 connecting to ZooKeeper ensemble=127.0.0.1:52803
2024-11-08T09:49:59,846 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,852 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,869 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:384410x0, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:49:59,870 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T09:49:59,868 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384410x0, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:49:59,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38441-0x10119da31030002 connected
2024-11-08T09:49:59,873 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T09:49:59,875 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:49:59,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:49:59,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38441
2024-11-08T09:49:59,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38441
2024-11-08T09:49:59,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38441
2024-11-08T09:49:59,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38441
2024-11-08T09:49:59,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38441
2024-11-08T09:49:59,910 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:49:59,910 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,910 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,911 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:49:59,911 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:49:59,911 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:49:59,911 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T09:49:59,912 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:49:59,913 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41283
2024-11-08T09:49:59,916 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41283 connecting to ZooKeeper ensemble=127.0.0.1:52803
2024-11-08T09:49:59,918 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,922 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:49:59,939 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412830x0, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:49:59,940 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T09:49:59,939 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412830x0, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:49:59,942 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41283-0x10119da31030003 connected
2024-11-08T09:49:59,944 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T09:49:59,945 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:49:59,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:49:59,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41283
2024-11-08T09:49:59,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41283
2024-11-08T09:49:59,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41283
2024-11-08T09:49:59,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41283
2024-11-08T09:49:59,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41283
2024-11-08T09:49:59,983 DEBUG [M:0;cf4d76213909:43985 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cf4d76213909:43985
2024-11-08T09:49:59,985 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cf4d76213909,43985,1731059398880
2024-11-08T09:50:00,002 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,002 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,008 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cf4d76213909,43985,1731059398880
2024-11-08T09:50:00,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,038 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,038 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,038 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,039 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,042 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-08T09:50:00,044 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cf4d76213909,43985,1731059398880 from backup master directory
2024-11-08T09:50:00,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cf4d76213909,43985,1731059398880
2024-11-08T09:50:00,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,057 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,058 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:00,059 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T09:50:00,060 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cf4d76213909,43985,1731059398880
2024-11-08T09:50:00,062 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-08T09:50:00,063 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-08T09:50:00,151 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/hbase.id] with ID: fc97fc20-75bd-4003-8a71-7d3bef83026d
2024-11-08T09:50:00,152 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/.tmp/hbase.id
2024-11-08T09:50:00,167 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'.
2024-11-08T09:50:00,167 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'.
2024-11-08T09:50:00,181 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_468708289_22 at /127.0.0.1:48886 [Receiving block BP-341519641-172.17.0.2-1731059393563:blk_-9223372036854775776_1003] {}] datanode.DataXceiver(331): 127.0.0.1:44789:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48886 dst: /127.0.0.1:44789
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:00,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44789 is added to blk_-9223372036854775776_1004 (size=42)
2024-11-08T09:50:00,595 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data.
2024-11-08T09:50:00,595 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/.tmp/hbase.id]:[hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/hbase.id]
2024-11-08T09:50:00,639 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:00,644 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-08T09:50:00,666 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms.
2024-11-08T09:50:00,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,717 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,717 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,729 WARN  [IPC Server handler 2 on default port 40093 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly.
2024-11-08T09:50:00,729 WARN  [IPC Server handler 2 on default port 40093 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:00,730 WARN  [IPC Server handler 2 on default port 40093 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-08T09:50:00,730 WARN  [IPC Server handler 2 on default port 40093 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-08T09:50:00,737 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] io.Closeables(82): IOException thrown while closing Closeable.
org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/.tmp/hbase-hbck.lock could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation.
	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480)
	at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) ~[hbase-shaded-miscellaneous-4.1.9.jar:4.1.9]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:994) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:00,759 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-08T09:50:00,761 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-08T09:50:00,767 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-11-08T09:50:00,795 WARN  [IPC Server handler 2 on default port 40093 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly.
2024-11-08T09:50:00,795 WARN  [IPC Server handler 2 on default port 40093 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:00,795 WARN  [IPC Server handler 2 on default port 40093 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-08T09:50:00,796 WARN  [IPC Server handler 2 on default port 40093 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-08T09:50:00,797 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189; retrying up to 10 times
org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation.
	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480)
	at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:00,804 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'.
2024-11-08T09:50:00,805 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'.
2024-11-08T09:50:00,814 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_468708289_22 at /127.0.0.1:53490 [Receiving block BP-341519641-172.17.0.2-1731059393563:blk_-9223372036854775760_1005] {}] datanode.DataXceiver(331): 127.0.0.1:35759:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53490 dst: /127.0.0.1:35759
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:00,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35759 is added to blk_-9223372036854775760_1006 (size=1189)
2024-11-08T09:50:00,821 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data.
2024-11-08T09:50:00,824 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189
2024-11-08T09:50:00,838 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/MasterData/data/master/store
2024-11-08T09:50:00,861 WARN  [IPC Server handler 4 on default port 40093 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly.
2024-11-08T09:50:00,861 WARN  [IPC Server handler 4 on default port 40093 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:00,861 WARN  [IPC Server handler 4 on default port 40093 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-08T09:50:00,861 WARN  [IPC Server handler 4 on default port 40093 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-08T09:50:00,863 ERROR [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(2539): Failed to become active master
org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.regioninfo could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation.
	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480)
	at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoFileContent(HRegionFileSystem.java:815) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:906) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:868) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:936) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7592) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:242) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:00,864 ERROR [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(3223): ***** ABORTING master cf4d76213909,43985,1731059398880: Unhandled exception. Starting shutdown. *****
org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.regioninfo could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation.
	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480)
	at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoFileContent(HRegionFileSystem.java:815) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:906) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.writeRegionInfoOnFilesystem(HRegionFileSystem.java:868) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:936) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:7592) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:242) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:00,866 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(3321): ***** STOPPING master 'cf4d76213909,43985,1731059398880' *****
2024-11-08T09:50:00,866 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(3323): STOPPED: Stopped by master/cf4d76213909:0:becomeActiveMaster
2024-11-08T09:50:00,866 INFO  [M:0;cf4d76213909:43985 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T09:50:00,866 INFO  [M:0;cf4d76213909:43985 {}] hbase.ChoreService(370): Chore service for: master/cf4d76213909:0 had [] on shutdown
2024-11-08T09:50:00,867 INFO  [M:0;cf4d76213909:43985 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T09:50:00,867 DEBUG [M:0;cf4d76213909:43985 {}] master.HMaster(1795): Stopping service threads
2024-11-08T09:50:00,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,878 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,879 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:00,879 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,880 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:00,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:50:00,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:50:00,889 DEBUG [M:0;cf4d76213909:43985 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-11-08T09:50:00,889 DEBUG [M:0;cf4d76213909:43985 {}] master.ActiveMasterManager(353): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-11-08T09:50:00,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:50:00,890 INFO  [M:0;cf4d76213909:43985 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43985
2024-11-08T09:50:00,893 INFO  [M:0;cf4d76213909:43985 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T09:50:01,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:01,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43985-0x10119da31030000, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:01,007 INFO  [M:0;cf4d76213909:43985 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T09:50:01,050 WARN  [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=2) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly.
2024-11-08T09:50:01,050 WARN  [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:01,051 WARN  [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=1) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly.
2024-11-08T09:50:01,051 WARN  [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:01,051 WARN  [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-08T09:50:01,051 WARN  [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable:  unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-08T09:50:03,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44789 is added to blk_-9223372036854775789_1002 (size=7)
2024-11-08T09:50:03,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41317 is added to blk_-9223372036854775757_1006 (size=1189)
2024-11-08T09:50:03,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44789 is added to blk_-9223372036854775756_1006 (size=1189)
2024-11-08T09:50:03,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41317 is added to blk_-9223372036854775773_1004 (size=42)
2024-11-08T09:50:03,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41317 is added to blk_-9223372036854775788_1002 (size=7)
2024-11-08T09:50:06,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35759 is added to blk_-9223372036854775772_1004 (size=42)
2024-11-08T09:50:27,676 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
Process Thread Dump: Thread dump because: Master not active after 30000ms
248 active threads
Thread 1 (main):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.FutureTask.awaitDone(FutureTask.java:444)
    java.base@17.0.11/java.util.concurrent.FutureTask.get(FutureTask.java:203)
    app//org.junit.internal.runners.statements.FailOnTimeout.getResult(FailOnTimeout.java:167)
    app//org.junit.internal.runners.statements.FailOnTimeout.evaluate(FailOnTimeout.java:128)
    app//org.apache.hadoop.hbase.SystemExitRule$1.evaluate(SystemExitRule.java:39)
    app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
    app//org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    app//org.junit.runners.ParentRunner.run(ParentRunner.java:413)
    app//org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316)
    app//org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240)
    app//org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214)
    app//org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155)
    app//org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
    app//org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
    app//org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
    app//org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)
Thread 2 (Reference Handler):
  State: RUNNABLE
  Blocked count: 1
  Waited count: 0
  Stack:
    java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method)
    java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253)
    java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215)
Thread 3 (Finalizer):
  State: WAITING
  Blocked count: 5
  Waited count: 6
  Waiting on java.lang.ref.ReferenceQueue$Lock@570c3b26
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
    java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172)
Thread 4 (Signal Dispatcher):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
Thread 12 (Common-Cleaner):
  State: TIMED_WAITING
  Blocked count: 9
  Waited count: 9
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
    java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
    java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162)
Thread 13 (Notification Thread):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
Thread 14 (pool-1-thread-1):
  State: RUNNABLE
  Blocked count: 2
  Waited count: 3
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.poll(EPollPort.java:200)
    java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:281)
    java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 15 (pool-1-thread-2):
  State: WAITING
  Blocked count: 0
  Waited count: 3
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6f818264
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
    java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:275)
    java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 16 (surefire-forkedjvm-stream-flusher):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 384
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 18 (surefire-forkedjvm-command-thread):
  State: WAITING
  Blocked count: 0
  Waited count: 5
  Waiting on java.util.concurrent.CountDownLatch$Sync@7762c340
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
    java.base@17.0.11/sun.nio.ch.PendingFuture.get(PendingFuture.java:178)
    app//org.apache.maven.surefire.api.util.internal.Channels$2.read(Channels.java:127)
    java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
    java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
    java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
    app//org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:169)
    app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:50)
    app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:430)
    app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:419)
    app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.readMessageType(AbstractStreamDecoder.java:116)
    app//org.apache.maven.surefire.booter.stream.CommandDecoder.decode(CommandDecoder.java:77)
    app//org.apache.maven.surefire.booter.spi.CommandChannelDecoder.decode(CommandChannelDecoder.java:60)
    app//org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:290)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 20 (process reaper):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 16
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 22 (Time-limited test):
  State: RUNNABLE
  Blocked count: 90
  Waited count: 416
  Stack:
    java.management@17.0.11/sun.management.ThreadImpl.getThreadInfo1(Native Method)
    java.management@17.0.11/sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:197)
    java.management@17.0.11/sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:154)
    app//org.apache.hadoop.hbase.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:181)
    app//org.apache.hadoop.hbase.util.Threads.printThreadInfo(Threads.java:186)
    app//org.apache.hadoop.hbase.util.JVMClusterUtil.waitForEvent(JVMClusterUtil.java:220)
    app//org.apache.hadoop.hbase.util.JVMClusterUtil.startup(JVMClusterUtil.java:177)
    app//org.apache.hadoop.hbase.LocalHBaseCluster.startup(LocalHBaseCluster.java:409)
    app//org.apache.hadoop.hbase.SingleProcessHBaseCluster.init(SingleProcessHBaseCluster.java:245)
    app//org.apache.hadoop.hbase.SingleProcessHBaseCluster.<init>(SingleProcessHBaseCluster.java:111)
    app//org.apache.hadoop.hbase.HBaseTestingUtil.startMiniHBaseCluster(HBaseTestingUtil.java:863)
    app//org.apache.hadoop.hbase.HBaseTestingUtil.startMiniCluster(HBaseTestingUtil.java:830)
    app//org.apache.hadoop.hbase.HBaseTestingUtil.startMiniCluster(HBaseTestingUtil.java:784)
    app//org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.setUp(TestHBaseWalOnEC.java:96)
    java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568)
    app//org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    app//org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
Thread 23 (org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.lang.ref.ReferenceQueue$Lock@40f16442
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
    app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 25 (SSL Certificates Store Monitor):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.TaskQueue@9933aec
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.lang.Object.wait(Object.java:338)
    java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
    java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
Thread 33 (Timer for 'NameNode' metrics system):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 5
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
    java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
Thread 34 (org.apache.hadoop.util.JvmPauseMonitor$Monitor@6d66e9d1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 72
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 35 (GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 8
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)
Thread 36 (pool-5-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 37 (qtp809254428-37):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 38 (qtp809254428-38):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 39 (qtp809254428-39):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 40 (qtp809254428-40):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 41 (qtp809254428-41-acceptor-0@e6afb64-ServerConnector@13e2962d{HTTP/1.1, (http/1.1)}{localhost:33309}):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
    app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 42 (qtp809254428-42):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 43 (qtp809254428-43):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 44 (qtp809254428-44):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 45 (Session-HouseKeeper-6ebba0c1-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 46 (pool-6-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 53 (FSEditLogAsync):
  State: WAITING
  Blocked count: 3
  Waited count: 48
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@40079fa
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
    app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:241)
    app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:250)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 55 (Socket Reader #1 for port 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
Thread 56 (IPC Server idle connection scanner for port 40093):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 5
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
    java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
Thread 58 (Hadoop-Metrics-Updater-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 61 (org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@10afbb16):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 62 (DatanodeAdminMonitor-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 12
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 49 (org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@40a21c70):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 7
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 47 (RedundancyMonitor):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 13
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    java.base@17.0.11/java.lang.Thread.sleep(Thread.java:344)
    java.base@17.0.11/java.util.concurrent.TimeUnit.sleep(TimeUnit.java:446)
    app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:5352)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 48 (MarkedDeleteBlockScrubberThread):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 3465
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$MarkedDeleteBlockScrubber.run(BlockManager.java:5326)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 51 (Block report processor):
  State: WAITING
  Blocked count: 0
  Waited count: 9
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@38428d58
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
    app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:5627)
    app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:5614)
Thread 57 (IPC Server Responder):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
    app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
Thread 54 (IPC Server listener on 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
Thread 64 (IPC Server handler 0 on default port 40093):
  State: TIMED_WAITING
  Blocked count: 18
  Waited count: 55
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 65 (IPC Server handler 1 on default port 40093):
  State: TIMED_WAITING
  Blocked count: 33
  Waited count: 55
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 66 (IPC Server handler 2 on default port 40093):
  State: TIMED_WAITING
  Blocked count: 17
  Waited count: 58
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 67 (IPC Server handler 3 on default port 40093):
  State: TIMED_WAITING
  Blocked count: 10
  Waited count: 65
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 68 (IPC Server handler 4 on default port 40093):
  State: TIMED_WAITING
  Blocked count: 7
  Waited count: 54
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 69 (pool-11-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 71 (org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@5098af1e):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 18
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 72 (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@73e38adb):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 7
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 73 (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@181b547a):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 74 (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@2eee5b49):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 75 (CacheReplicationMonitor(1944697449)):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759)
    app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186)
Thread 76 (StorageLocationChecker thread 0):
  State: TIMED_WAITING
  Blocked count: 13
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 77 (StorageLocationChecker thread 1):
  State: TIMED_WAITING
  Blocked count: 5
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 78 (process reaper):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 10
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 86 (pool-17-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 87 (qtp240740103-87):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 88 (qtp240740103-88-acceptor-0@6fd4efaa-ServerConnector@383014b{HTTP/1.1, (http/1.1)}{localhost:41431}):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
    app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 89 (qtp240740103-89):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 90 (qtp240740103-90):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 91 (Session-HouseKeeper-e9c9ee1-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 92 (nioEventLoopGroup-2-1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
    app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
    app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
    app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
    app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 93 (org.apache.hadoop.util.JvmPauseMonitor$Monitor@3d57971c):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 70
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 95 (Socket Reader #1 for port 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
Thread 96 (IPC Server idle connection scanner for port 37075):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 5
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
    java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
Thread 98 (Hadoop-Metrics-Updater-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 101 (Command processor):
  State: WAITING
  Blocked count: 13
  Waited count: 13
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6909540b
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)
Thread 102 (BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093):
  State: TIMED_WAITING
  Blocked count: 152
  Waited count: 43
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 103 (pool-19-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 85 (org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@62763aa7):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126)
    app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85)
    app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 97 (IPC Server Responder):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
    app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
Thread 94 (IPC Server listener on 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
Thread 104 (IPC Server handler 0 on default port 37075):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 45
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 105 (IPC Server handler 1 on default port 37075):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 40
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 106 (IPC Server handler 2 on default port 37075):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 43
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 107 (IPC Server handler 3 on default port 37075):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 35
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 108 (IPC Server handler 4 on default port 37075):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 35
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 109 (StorageLocationChecker thread 0):
  State: TIMED_WAITING
  Blocked count: 4
  Waited count: 4
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 110 (StorageLocationChecker thread 1):
  State: TIMED_WAITING
  Blocked count: 4
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 118 (pool-25-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 119 (qtp1973657871-119):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 120 (qtp1973657871-120-acceptor-0@66c24284-ServerConnector@29a123ec{HTTP/1.1, (http/1.1)}{localhost:40661}):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
    app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 121 (qtp1973657871-121):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 122 (qtp1973657871-122):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 123 (Session-HouseKeeper-7bbf7334-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 124 (IPC Client (1462372341) connection to localhost/127.0.0.1:40093 from jenkins):
  State: TIMED_WAITING
  Blocked count: 34
  Waited count: 35
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
    app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
Thread 125 (IPC Parameter Sending Thread for localhost/127.0.0.1:40093):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 38
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 126 (nioEventLoopGroup-4-1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
    app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
    app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
    app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
    app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 127 (org.apache.hadoop.util.JvmPauseMonitor$Monitor@1e18ae41):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 69
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 129 (Socket Reader #1 for port 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
Thread 130 (IPC Server idle connection scanner for port 46105):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 5
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
    java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
Thread 132 (Hadoop-Metrics-Updater-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 135 (Command processor):
  State: WAITING
  Blocked count: 0
  Waited count: 13
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23e177ba
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)
Thread 136 (BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093):
  State: TIMED_WAITING
  Blocked count: 191
  Waited count: 41
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 137 (pool-28-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 117 (org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@7f5075f):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126)
    app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85)
    app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 131 (IPC Server Responder):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
    app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
Thread 128 (IPC Server listener on 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
Thread 138 (IPC Server handler 0 on default port 46105):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 35
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 139 (IPC Server handler 1 on default port 46105):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 35
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 140 (IPC Server handler 2 on default port 46105):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 35
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 141 (IPC Server handler 3 on default port 46105):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 36
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 142 (IPC Server handler 4 on default port 46105):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 35
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 143 (StorageLocationChecker thread 0):
  State: TIMED_WAITING
  Blocked count: 2
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 144 (StorageLocationChecker thread 1):
  State: TIMED_WAITING
  Blocked count: 3
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 152 (pool-35-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 153 (qtp2014619351-153):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
    app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
    app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
    app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
    app//org.eclipse.jetty.io.ManagedSelector$$Lambda$255/0x00007fb220443a50.run(Unknown Source)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 154 (qtp2014619351-154-acceptor-0@41ac7edb-ServerConnector@404caff2{HTTP/1.1, (http/1.1)}{localhost:40789}):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
    app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 155 (qtp2014619351-155):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 156 (qtp2014619351-156):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
    app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 157 (Session-HouseKeeper-67dddf34-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 158 (nioEventLoopGroup-6-1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
    app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
    app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
    app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
    app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 159 (org.apache.hadoop.util.JvmPauseMonitor$Monitor@7dbb853):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 69
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 161 (Socket Reader #1 for port 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497)
    app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476)
Thread 162 (IPC Server idle connection scanner for port 40399):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 5
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
    java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
Thread 164 (Hadoop-Metrics-Updater-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 167 (Command processor):
  State: WAITING
  Blocked count: 0
  Waited count: 13
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@28e1c4ae
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)
Thread 168 (BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093):
  State: TIMED_WAITING
  Blocked count: 27
  Waited count: 37
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771)
    app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 169 (pool-37-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 151 (org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@7e130936):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.Net.accept(Native Method)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
    java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
    java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126)
    app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85)
    app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 163 (IPC Server Responder):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733)
    app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716)
Thread 160 (IPC Server listener on 0):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559)
Thread 170 (IPC Server handler 0 on default port 40399):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 45
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 171 (IPC Server handler 1 on default port 40399):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 47
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 172 (IPC Server handler 2 on default port 40399):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 47
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 173 (IPC Server handler 3 on default port 40399):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 36
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 174 (IPC Server handler 4 on default port 40399):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 41
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370)
    app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165)
Thread 185 (VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data3)):
  State: TIMED_WAITING
  Blocked count: 30
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
Thread 186 (VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data4)):
  State: TIMED_WAITING
  Blocked count: 25
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
Thread 189 (VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data1)):
  State: TIMED_WAITING
  Blocked count: 28
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
Thread 190 (VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data2)):
  State: TIMED_WAITING
  Blocked count: 24
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
Thread 195 (process reaper):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 197 (refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data4/current/BP-341519641-172.17.0.2-1731059393563):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 198 (refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data3/current/BP-341519641-172.17.0.2-1731059393563):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 200 (refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data2/current/BP-341519641-172.17.0.2-1731059393563):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 201 (refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data1/current/BP-341519641-172.17.0.2-1731059393563):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 204 (ForkJoinPool-2-worker-1):
  State: WAITING
  Blocked count: 0
  Waited count: 3
  Waiting on java.util.concurrent.ForkJoinPool@1ea544b3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
    java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)
Thread 205 (ForkJoinPool-2-worker-2):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 4
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
    java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)
Thread 206 (ForkJoinPool-2-worker-3):
  State: WAITING
  Blocked count: 0
  Waited count: 2
  Waiting on java.util.concurrent.ForkJoinPool@1ea544b3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
    java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)
Thread 207 (ForkJoinPool-2-worker-4):
  State: WAITING
  Blocked count: 0
  Waited count: 3
  Waiting on java.util.concurrent.ForkJoinPool@1ea544b3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
    java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)
Thread 208 (DataNode DiskChecker thread 0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 211 (pool-14-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 212 (DataNode DiskChecker thread 0):
  State: TIMED_WAITING
  Blocked count: 3
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 213 (pool-22-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 218 (java.util.concurrent.ThreadPoolExecutor$Worker@5553daa6[State = -1, empty queue]):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 219 (java.util.concurrent.ThreadPoolExecutor$Worker@3c6b6376[State = -1, empty queue]):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 220 (VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data5)):
  State: TIMED_WAITING
  Blocked count: 2
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
Thread 221 (VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data6)):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656)
Thread 226 (refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data6/current/BP-341519641-172.17.0.2-1731059393563):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 227 (refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data5/current/BP-341519641-172.17.0.2-1731059393563):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 230 (DataNode DiskChecker thread 0):
  State: TIMED_WAITING
  Blocked count: 1
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 231 (pool-32-thread-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 232 (DataNode DiskChecker thread 1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 235 (java.util.concurrent.ThreadPoolExecutor$Worker@3a81af8e[State = -1, empty queue]):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 236 (FsDatasetAsyncDiskServiceFixer):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 2
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtil.java:576)
Thread 242 (LeaseRenewer:jenkins@localhost:40093):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 33
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
    app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
    app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 249 (NIOServerCxnFactory.SelectorThread-0):
  State: RUNNABLE
  Blocked count: 1
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
Thread 250 (NIOServerCxnFactory.SelectorThread-1):
  State: RUNNABLE
  Blocked count: 1
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
Thread 251 (NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:52803):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
Thread 248 (ConnnectionExpirer):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 4
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)
Thread 252 (SessionTracker):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 17
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)
Thread 253 (SyncThread:0):
  State: WAITING
  Blocked count: 0
  Waited count: 60
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@593f2c38
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
Thread 254 (ProcessThread(sid:0 cport:52803):):
  State: WAITING
  Blocked count: 0
  Waited count: 77
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@766a684e
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
Thread 255 (RequestThrottler):
  State: WAITING
  Blocked count: 0
  Waited count: 81
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5b7f544a
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
Thread 256 (NIOWorkerThread-1):
  State: WAITING
  Blocked count: 0
  Waited count: 13
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 272 (weak-ref-cleaner-strictcontextstorage):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.lang.ref.ReferenceQueue$Lock@43cff55c
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
    java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
    app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 273 (HBase-Metrics2-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 4
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 274 (HMaster-EventLoopGroup-1-1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
    app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 277 (NIOWorkerThread-2):
  State: WAITING
  Blocked count: 0
  Waited count: 13
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 278 (NIOWorkerThread-3):
  State: WAITING
  Blocked count: 0
  Waited count: 13
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 279 (NIOWorkerThread-4):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 281 (NIOWorkerThread-5):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 282 (NIOWorkerThread-6):
  State: WAITING
  Blocked count: 1
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 283 (NIOWorkerThread-7):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 284 (NIOWorkerThread-8):
  State: WAITING
  Blocked count: 1
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 285 (NIOWorkerThread-9):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 286 (NIOWorkerThread-10):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 287 (NIOWorkerThread-11):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 288 (NIOWorkerThread-12):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 289 (NIOWorkerThread-13):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 290 (NIOWorkerThread-14):
  State: WAITING
  Blocked count: 1
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 291 (NIOWorkerThread-15):
  State: WAITING
  Blocked count: 1
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 292 (NIOWorkerThread-16):
  State: WAITING
  Blocked count: 0
  Waited count: 12
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@63315bee
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 306 (Time-limited test.named-queue-events-pool-0):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@473e18e5
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
    app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
    app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
    app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 307 (MiniHBaseClusterRegionServer-EventLoopGroup-3-1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
    app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 309 (Time-limited test-SendThread(127.0.0.1:52803)):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
    app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
Thread 310 (Time-limited test-EventThread):
  State: WAITING
  Blocked count: 2
  Waited count: 9
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2eb9d5c7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
Thread 311 (zk-event-processor-pool-0):
  State: WAITING
  Blocked count: 8
  Waited count: 14
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@12e0a059
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 312 (Time-limited test.LruBlockCache.EvictionThread):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 4
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957)
Thread 314 (LruBlockCacheStatsExecutor):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 316 (MobFileCache #0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 317 (RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@2275f7b4
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 318 (RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@a157d9f
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 319 (RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@63d6d57f
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 320 (RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@55b943e3
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 321 (RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@58fcc6b7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 322 (RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@58fcc6b7
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 323 (RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@7924b1b2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 324 (RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@2c0ceec4
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 325 (RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@33b516fd
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 326 (RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=42175):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@dc2a742
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 329 (MiniHBaseClusterRegionServer-EventLoopGroup-4-1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
    app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 331 (pool-71-thread-1-SendThread(127.0.0.1:52803)):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
    app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
Thread 332 (pool-71-thread-1-EventThread):
  State: WAITING
  Blocked count: 5
  Waited count: 8
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@51373e69
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
Thread 333 (Time-limited test.LruBlockCache.EvictionThread):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 4
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957)
Thread 335 (LruBlockCacheStatsExecutor):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 336 (zk-event-processor-pool-0):
  State: WAITING
  Blocked count: 11
  Waited count: 14
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@19ff30c0
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 338 (MobFileCache #0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 339 (RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@5f79b337
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 340 (RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@25e49091
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 341 (RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@45fdcddd
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 342 (RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@284c9813
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 343 (RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3a3e6626
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 344 (RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3a3e6626
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 345 (RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@3bdf19e8
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 346 (RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@1067b86b
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 347 (RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@5c272674
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 348 (RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=38441):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@15e17480
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 351 (MiniHBaseClusterRegionServer-EventLoopGroup-5-1):
  State: RUNNABLE
  Blocked count: 1
  Waited count: 0
  Stack:
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
    app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
    app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 353 (pool-77-thread-1-SendThread(127.0.0.1:52803)):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
    java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
    java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
    app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
    app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289)
Thread 354 (pool-77-thread-1-EventThread):
  State: WAITING
  Blocked count: 1
  Waited count: 7
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@56b22f8d
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550)
Thread 355 (Time-limited test.LruBlockCache.EvictionThread):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 4
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957)
Thread 356 (zk-event-processor-pool-0):
  State: WAITING
  Blocked count: 9
  Waited count: 14
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ecaec55
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 358 (LruBlockCacheStatsExecutor):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 360 (MobFileCache #0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 361 (RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@670d9b62
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 362 (RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@2eec7b92
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 363 (RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@20008229
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 364 (RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@25a0e45e
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 365 (RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e7dbd03
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 366 (RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e7dbd03
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
    java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:68)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 367 (RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@630063b5
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 368 (RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@4735b436
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 369 (RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@65952a9a
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 370 (RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=41283):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on java.util.concurrent.Semaphore$NonfairSync@26c3168e
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
    java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318)
    app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55)
    app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)
Thread 374 (Monitor thread for TaskMonitor):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 3
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 376 (master/cf4d76213909:0:becomeActiveMaster-MemStoreChunkPool Statistics):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 378 (master/cf4d76213909:0:becomeActiveMaster-MemStoreChunkPool Statistics):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
    java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 380 (StripedRead-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 381 (org.apache.hadoop.hdfs.PeerCache@25e51626):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 10
  Stack:
    java.base@17.0.11/java.lang.Thread.sleep(Native Method)
    app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
    app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
    app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 403 (master:store-WAL-Roller):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 293
  Stack:
    java.base@17.0.11/java.lang.Object.wait(Native Method)
    app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180)
Thread 429 (StripedBlockReconstruction-0):
  State: TIMED_WAITING
  Blocked count: 11
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 428 (StripedBlockReconstruction-0):
  State: TIMED_WAITING
  Blocked count: 7
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 430 (StripedBlockReconstruction-1):
  State: TIMED_WAITING
  Blocked count: 12
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 440 (stripedRead-1):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 439 (stripedRead-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 441 (stripedRead-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 442 (StripedBlockReconstruction-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 2
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
    java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
Thread 445 (stripedRead-0):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 1
  Stack:
    java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
    java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
    java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
    java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
    java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
2024-11-08T09:50:30,025 ERROR [Time-limited test {}] hbase.SingleProcessHBaseCluster(250): Error starting cluster
java.lang.RuntimeException: Master not active after 30000ms
	at org.apache.hadoop.hbase.util.JVMClusterUtil.waitForEvent(JVMClusterUtil.java:221) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.JVMClusterUtil.startup(JVMClusterUtil.java:177) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.LocalHBaseCluster.startup(LocalHBaseCluster.java:409) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.SingleProcessHBaseCluster.init(SingleProcessHBaseCluster.java:245) ~[test-classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.SingleProcessHBaseCluster.<init>(SingleProcessHBaseCluster.java:111) ~[test-classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.HBaseTestingUtil.startMiniHBaseCluster(HBaseTestingUtil.java:863) ~[test-classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.HBaseTestingUtil.startMiniCluster(HBaseTestingUtil.java:830) ~[test-classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.HBaseTestingUtil.startMiniCluster(HBaseTestingUtil.java:784) ~[test-classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.setUp(TestHBaseWalOnEC.java:96) ~[test-classes/:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-08T09:50:30,025 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,42175,1731059399709' *****
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,38441,1731059399838' *****
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,41283,1731059399909' *****
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T09:50:30,026 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete
2024-11-08T09:50:30,027 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-08T09:50:30,090 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@653e6301{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:30,094 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@404caff2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:30,095 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:30,095 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@343b36c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:30,095 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35e2f174{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:30,099 WARN  [BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T09:50:30,099 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T09:50:30,100 WARN  [BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-341519641-172.17.0.2-1731059393563 (Datanode Uuid 12abefdb-119b-4b5d-9c26-679a1cd1b111) service to localhost/127.0.0.1:40093
2024-11-08T09:50:30,100 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T09:50:30,101 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data5/current/BP-341519641-172.17.0.2-1731059393563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:30,101 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data6/current/BP-341519641-172.17.0.2-1731059393563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:30,102 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T09:50:30,104 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@513cab2c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:30,105 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29a123ec{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:30,105 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:30,105 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6af5a446{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:30,106 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444b27d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:30,107 WARN  [BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T09:50:30,107 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T09:50:30,107 WARN  [BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-341519641-172.17.0.2-1731059393563 (Datanode Uuid 158f7428-aeb0-4ee5-93e8-ca384654f533) service to localhost/127.0.0.1:40093
2024-11-08T09:50:30,108 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T09:50:30,108 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data3/current/BP-341519641-172.17.0.2-1731059393563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:30,108 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data4/current/BP-341519641-172.17.0.2-1731059393563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:30,109 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T09:50:30,111 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65462677{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:30,111 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@383014b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:30,112 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:30,112 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4dc262e0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:30,112 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@431e53b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:30,114 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T09:50:30,114 WARN  [BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T09:50:30,114 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T09:50:30,114 WARN  [BP-341519641-172.17.0.2-1731059393563 heartbeating to localhost/127.0.0.1:40093 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-341519641-172.17.0.2-1731059393563 (Datanode Uuid c8a235e3-00ab-417c-a743-4d2342d5a572) service to localhost/127.0.0.1:40093
2024-11-08T09:50:30,114 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data1/current/BP-341519641-172.17.0.2-1731059393563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:30,115 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/cluster_9a80b8c5-8d62-fe7e-5e5b-8517ccba7f6a/data/data2/current/BP-341519641-172.17.0.2-1731059393563 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:30,115 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T09:50:30,125 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58dbf239{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T09:50:30,126 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13e2962d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:30,126 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:30,126 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f93dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:30,126 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4395d44b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:30,133 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-08T09:50:30,159 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-08T09:50:30,169 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestHBaseWalOnEC#testReadWrite[0] Thread=105 (was 159), OpenFileDescriptor=374 (was 393), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 225) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4719 (was 5776)
2024-11-08T09:50:30,175 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestHBaseWalOnEC#testReadWrite[1] Thread=105, OpenFileDescriptor=374, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=4719
2024-11-08T09:50:30,175 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-08T09:50:30,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.log.dir so I do NOT create it in target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13
2024-11-08T09:50:30,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/1572d9e0-7bd5-ab40-4f42-8b4e2d1ad106/hadoop.tmp.dir so I do NOT create it in target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13
2024-11-08T09:50:30,176 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043, deleteOnExit=true
2024-11-08T09:50:30,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-08T09:50:30,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/test.cache.data in system properties and HBase conf
2024-11-08T09:50:30,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.tmp.dir in system properties and HBase conf
2024-11-08T09:50:30,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir in system properties and HBase conf
2024-11-08T09:50:30,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-08T09:50:30,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-08T09:50:30,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-08T09:50:30,177 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-08T09:50:30,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-08T09:50:30,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-08T09:50:30,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-08T09:50:30,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T09:50:30,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-08T09:50:30,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-08T09:50:30,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-08T09:50:30,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T09:50:30,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-08T09:50:30,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/nfs.dump.dir in system properties and HBase conf
2024-11-08T09:50:30,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/java.io.tmpdir in system properties and HBase conf
2024-11-08T09:50:30,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-08T09:50:30,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-08T09:50:30,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-08T09:50:30,231 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null
2024-11-08T09:50:30,231 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null
2024-11-08T09:50:30,231 DEBUG [pool-77-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null
2024-11-08T09:50:30,231 DEBUG [pool-71-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null
2024-11-08T09:50:30,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring
2024-11-08T09:50:30,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring
2024-11-08T09:50:30,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null
2024-11-08T09:50:30,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null
2024-11-08T09:50:30,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:38441-0x10119da31030002, quorum=127.0.0.1:52803, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring
2024-11-08T09:50:30,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:41283-0x10119da31030003, quorum=127.0.0.1:52803, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring
2024-11-08T09:50:30,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring
2024-11-08T09:50:30,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(640): regionserver:42175-0x10119da31030001, quorum=127.0.0.1:52803, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring
2024-11-08T09:50:30,578 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:50:30,584 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:50:30,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:50:30,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:50:30,587 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-08T09:50:30,588 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:50:30,588 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1120ea6a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:50:30,589 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dabbb4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:50:30,700 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b02112e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/java.io.tmpdir/jetty-localhost-38627-hadoop-hdfs-3_4_1-tests_jar-_-any-4361621346258702937/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T09:50:30,701 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@365b31e8{HTTP/1.1, (http/1.1)}{localhost:38627}
2024-11-08T09:50:30,701 INFO  [Time-limited test {}] server.Server(415): Started @39509ms
2024-11-08T09:50:30,973 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:50:30,978 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:50:30,979 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:50:30,979 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:50:30,979 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-08T09:50:30,980 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1592a7cf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:50:30,980 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@447c571b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:50:31,083 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3677c717{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/java.io.tmpdir/jetty-localhost-36311-hadoop-hdfs-3_4_1-tests_jar-_-any-5384745005254349867/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:31,083 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11762ce6{HTTP/1.1, (http/1.1)}{localhost:36311}
2024-11-08T09:50:31,083 INFO  [Time-limited test {}] server.Server(415): Started @39891ms
2024-11-08T09:50:31,086 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T09:50:31,145 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:50:31,153 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:50:31,154 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:50:31,154 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:50:31,154 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T09:50:31,155 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f624b91{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:50:31,155 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@128490e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:50:31,272 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@151f492e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/java.io.tmpdir/jetty-localhost-45245-hadoop-hdfs-3_4_1-tests_jar-_-any-12511054260863163948/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:31,273 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@ea7545b{HTTP/1.1, (http/1.1)}{localhost:45245}
2024-11-08T09:50:31,273 INFO  [Time-limited test {}] server.Server(415): Started @40081ms
2024-11-08T09:50:31,275 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T09:50:31,348 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-08T09:50:31,354 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-08T09:50:31,356 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-08T09:50:31,356 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-08T09:50:31,356 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-08T09:50:31,357 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c6a742f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,AVAILABLE}
2024-11-08T09:50:31,358 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@243a1ac8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-08T09:50:31,492 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d404ea1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/java.io.tmpdir/jetty-localhost-42649-hadoop-hdfs-3_4_1-tests_jar-_-any-15723324451992175569/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:31,493 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6732f353{HTTP/1.1, (http/1.1)}{localhost:42649}
2024-11-08T09:50:31,493 INFO  [Time-limited test {}] server.Server(415): Started @40301ms
2024-11-08T09:50:31,496 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-08T09:50:32,326 WARN  [Thread-342 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data2/current/BP-1316722941-172.17.0.2-1731059430203/current, will proceed with Du for space computation calculation, 
2024-11-08T09:50:32,326 WARN  [Thread-341 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data1/current/BP-1316722941-172.17.0.2-1731059430203/current, will proceed with Du for space computation calculation, 
2024-11-08T09:50:32,345 WARN  [Thread-282 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T09:50:32,351 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x231c4f9ff6888abe with lease ID 0x758aa7ee15760f6d: Processing first storage report for DS-d7c05715-b8e1-41dc-9c0a-185ecab43442 from datanode DatanodeRegistration(127.0.0.1:43647, datanodeUuid=1a8191e9-7592-4fa1-9252-160b3f1fe7e6, infoPort=36767, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203)
2024-11-08T09:50:32,351 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x231c4f9ff6888abe with lease ID 0x758aa7ee15760f6d: from storage DS-d7c05715-b8e1-41dc-9c0a-185ecab43442 node DatanodeRegistration(127.0.0.1:43647, datanodeUuid=1a8191e9-7592-4fa1-9252-160b3f1fe7e6, infoPort=36767, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:50:32,352 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x231c4f9ff6888abe with lease ID 0x758aa7ee15760f6d: Processing first storage report for DS-b0d754a8-0aa1-4408-8685-41a76818d7b7 from datanode DatanodeRegistration(127.0.0.1:43647, datanodeUuid=1a8191e9-7592-4fa1-9252-160b3f1fe7e6, infoPort=36767, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203)
2024-11-08T09:50:32,352 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x231c4f9ff6888abe with lease ID 0x758aa7ee15760f6d: from storage DS-b0d754a8-0aa1-4408-8685-41a76818d7b7 node DatanodeRegistration(127.0.0.1:43647, datanodeUuid=1a8191e9-7592-4fa1-9252-160b3f1fe7e6, infoPort=36767, infoSecurePort=0, ipcPort=38521, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:50:32,566 WARN  [Thread-354 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data4/current/BP-1316722941-172.17.0.2-1731059430203/current, will proceed with Du for space computation calculation, 
2024-11-08T09:50:32,566 WARN  [Thread-353 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data3/current/BP-1316722941-172.17.0.2-1731059430203/current, will proceed with Du for space computation calculation, 
2024-11-08T09:50:32,585 WARN  [Thread-305 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T09:50:32,588 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e3c0bec08d78615 with lease ID 0x758aa7ee15760f6e: Processing first storage report for DS-c739a744-ea5b-4f93-9bca-b0f369be74b1 from datanode DatanodeRegistration(127.0.0.1:37415, datanodeUuid=a2953b2b-8d80-48a1-bf02-cef6acedcb8b, infoPort=44491, infoSecurePort=0, ipcPort=45529, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203)
2024-11-08T09:50:32,589 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e3c0bec08d78615 with lease ID 0x758aa7ee15760f6e: from storage DS-c739a744-ea5b-4f93-9bca-b0f369be74b1 node DatanodeRegistration(127.0.0.1:37415, datanodeUuid=a2953b2b-8d80-48a1-bf02-cef6acedcb8b, infoPort=44491, infoSecurePort=0, ipcPort=45529, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:50:32,589 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e3c0bec08d78615 with lease ID 0x758aa7ee15760f6e: Processing first storage report for DS-7700aebe-858a-458d-8e91-dd10dd561a5d from datanode DatanodeRegistration(127.0.0.1:37415, datanodeUuid=a2953b2b-8d80-48a1-bf02-cef6acedcb8b, infoPort=44491, infoSecurePort=0, ipcPort=45529, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203)
2024-11-08T09:50:32,589 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e3c0bec08d78615 with lease ID 0x758aa7ee15760f6e: from storage DS-7700aebe-858a-458d-8e91-dd10dd561a5d node DatanodeRegistration(127.0.0.1:37415, datanodeUuid=a2953b2b-8d80-48a1-bf02-cef6acedcb8b, infoPort=44491, infoSecurePort=0, ipcPort=45529, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:50:32,737 WARN  [Thread-365 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data6/current/BP-1316722941-172.17.0.2-1731059430203/current, will proceed with Du for space computation calculation, 
2024-11-08T09:50:32,737 WARN  [Thread-364 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data5/current/BP-1316722941-172.17.0.2-1731059430203/current, will proceed with Du for space computation calculation, 
2024-11-08T09:50:32,760 WARN  [Thread-327 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-08T09:50:32,763 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd79c32fe6746d151 with lease ID 0x758aa7ee15760f6f: Processing first storage report for DS-48962ff0-9b26-47ae-952c-b3a30dd28cdb from datanode DatanodeRegistration(127.0.0.1:34075, datanodeUuid=4de35748-0b39-4dfe-817e-08e27beb61d8, infoPort=46277, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203)
2024-11-08T09:50:32,763 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd79c32fe6746d151 with lease ID 0x758aa7ee15760f6f: from storage DS-48962ff0-9b26-47ae-952c-b3a30dd28cdb node DatanodeRegistration(127.0.0.1:34075, datanodeUuid=4de35748-0b39-4dfe-817e-08e27beb61d8, infoPort=46277, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:50:32,763 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd79c32fe6746d151 with lease ID 0x758aa7ee15760f6f: Processing first storage report for DS-0f1dd5bd-19bc-43c1-9459-005fdd910afa from datanode DatanodeRegistration(127.0.0.1:34075, datanodeUuid=4de35748-0b39-4dfe-817e-08e27beb61d8, infoPort=46277, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203)
2024-11-08T09:50:32,763 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd79c32fe6746d151 with lease ID 0x758aa7ee15760f6f: from storage DS-0f1dd5bd-19bc-43c1-9459-005fdd910afa node DatanodeRegistration(127.0.0.1:34075, datanodeUuid=4de35748-0b39-4dfe-817e-08e27beb61d8, infoPort=46277, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=1343043469;c=1731059430203), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-08T09:50:32,858 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13
2024-11-08T09:50:32,861 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/zookeeper_0, clientPort=59691, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-08T09:50:32,863 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59691
2024-11-08T09:50:32,863 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:32,865 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:32,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741825_1001 (size=7)
2024-11-08T09:50:32,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741825_1001 (size=7)
2024-11-08T09:50:32,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741825_1001 (size=7)
2024-11-08T09:50:32,885 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3 with version=8
2024-11-08T09:50:32,885 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40093/user/jenkins/test-data/9ddb0ace-d46d-a480-76dc-c0a2da33a280/hbase-staging
2024-11-08T09:50:32,888 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:50:32,888 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:32,888 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:32,888 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:50:32,889 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:32,889 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:50:32,889 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-08T09:50:32,889 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:50:32,890 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45843
2024-11-08T09:50:32,892 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45843 connecting to ZooKeeper ensemble=127.0.0.1:59691
2024-11-08T09:50:33,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:458430x0, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:50:33,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45843-0x10119dab9480000 connected
2024-11-08T09:50:33,214 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,215 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:33,219 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3, hbase.cluster.distributed=false
2024-11-08T09:50:33,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:50:33,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45843
2024-11-08T09:50:33,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45843
2024-11-08T09:50:33,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45843
2024-11-08T09:50:33,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45843
2024-11-08T09:50:33,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45843
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T09:50:33,239 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:50:33,240 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40561
2024-11-08T09:50:33,242 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40561 connecting to ZooKeeper ensemble=127.0.0.1:59691
2024-11-08T09:50:33,243 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,245 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,255 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405610x0, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:50:33,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40561-0x10119dab9480001 connected
2024-11-08T09:50:33,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:33,257 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T09:50:33,259 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T09:50:33,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:50:33,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:50:33,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40561
2024-11-08T09:50:33,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40561
2024-11-08T09:50:33,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40561
2024-11-08T09:50:33,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40561
2024-11-08T09:50:33,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40561
2024-11-08T09:50:33,286 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:50:33,286 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,287 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,287 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:50:33,287 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,287 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:50:33,287 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T09:50:33,287 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:50:33,288 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36537
2024-11-08T09:50:33,289 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36537 connecting to ZooKeeper ensemble=127.0.0.1:59691
2024-11-08T09:50:33,290 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,292 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365370x0, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:50:33,309 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365370x0, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:33,309 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36537-0x10119dab9480002 connected
2024-11-08T09:50:33,309 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T09:50:33,310 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T09:50:33,311 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:50:33,312 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:50:33,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36537
2024-11-08T09:50:33,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36537
2024-11-08T09:50:33,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36537
2024-11-08T09:50:33,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36537
2024-11-08T09:50:33,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36537
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-08T09:50:33,357 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-08T09:50:33,358 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-08T09:50:33,359 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38083
2024-11-08T09:50:33,362 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38083 connecting to ZooKeeper ensemble=127.0.0.1:59691
2024-11-08T09:50:33,363 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,366 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380830x0, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-08T09:50:33,382 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380830x0, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:33,383 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38083-0x10119dab9480003 connected
2024-11-08T09:50:33,383 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-08T09:50:33,384 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-08T09:50:33,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-08T09:50:33,386 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-08T09:50:33,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38083
2024-11-08T09:50:33,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38083
2024-11-08T09:50:33,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38083
2024-11-08T09:50:33,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38083
2024-11-08T09:50:33,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38083
2024-11-08T09:50:33,399 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cf4d76213909:45843
2024-11-08T09:50:33,400 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cf4d76213909,45843,1731059432888
2024-11-08T09:50:33,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,413 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,414 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cf4d76213909,45843,1731059432888
2024-11-08T09:50:33,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:33,424 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:33,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:33,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,424 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,425 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-08T09:50:33,425 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cf4d76213909,45843,1731059432888 from backup master directory
2024-11-08T09:50:33,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cf4d76213909,45843,1731059432888
2024-11-08T09:50:33,435 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,435 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T09:50:33,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-08T09:50:33,435 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cf4d76213909,45843,1731059432888
2024-11-08T09:50:33,443 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/hbase.id] with ID: bf0214c8-0430-400d-8f92-2fc658fea88f
2024-11-08T09:50:33,443 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/.tmp/hbase.id
2024-11-08T09:50:33,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741826_1002 (size=42)
2024-11-08T09:50:33,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741826_1002 (size=42)
2024-11-08T09:50:33,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741826_1002 (size=42)
2024-11-08T09:50:33,456 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/.tmp/hbase.id]:[hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/hbase.id]
2024-11-08T09:50:33,481 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-08T09:50:33,481 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-08T09:50:33,483 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-08T09:50:33,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,558 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:33,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741827_1003 (size=196)
2024-11-08T09:50:33,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741827_1003 (size=196)
2024-11-08T09:50:33,575 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741827_1003 (size=196)
2024-11-08T09:50:33,576 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-08T09:50:33,578 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-08T09:50:33,578 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-08T09:50:33,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741828_1004 (size=1189)
2024-11-08T09:50:33,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741828_1004 (size=1189)
2024-11-08T09:50:33,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741828_1004 (size=1189)
2024-11-08T09:50:33,595 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store
2024-11-08T09:50:33,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741829_1005 (size=34)
2024-11-08T09:50:33,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741829_1005 (size=34)
2024-11-08T09:50:33,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741829_1005 (size=34)
2024-11-08T09:50:33,611 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-08T09:50:33,614 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T09:50:33,615 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-08T09:50:33,615 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:33,615 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:33,616 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-08T09:50:33,616 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:33,616 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:33,616 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731059433615Disabling compacts and flushes for region at 1731059433615Disabling writes for close at 1731059433616 (+1 ms)Writing region close event to WAL at 1731059433616Closed at 1731059433616
2024-11-08T09:50:33,618 WARN  [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/.initializing
2024-11-08T09:50:33,618 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/WALs/cf4d76213909,45843,1731059432888
2024-11-08T09:50:33,641 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C45843%2C1731059432888, suffix=, logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/WALs/cf4d76213909,45843,1731059432888, archiveDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/oldWALs, maxLogs=10
2024-11-08T09:50:33,652 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor cf4d76213909%2C45843%2C1731059432888.1731059433646
2024-11-08T09:50:33,675 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/WALs/cf4d76213909,45843,1731059432888/cf4d76213909%2C45843%2C1731059432888.1731059433646
2024-11-08T09:50:33,685 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36767:36767),(127.0.0.1/127.0.0.1:46277:46277),(127.0.0.1/127.0.0.1:44491:44491)]
2024-11-08T09:50:33,686 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-08T09:50:33,687 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T09:50:33,690 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,691 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,725 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T09:50:33,748 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-08T09:50:33,752 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:33,755 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:33,755 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T09:50:33,757 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-08T09:50:33,758 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:33,759 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T09:50:33,759 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T09:50:33,761 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-08T09:50:33,761 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:33,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T09:50:33,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-08T09:50:33,764 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-08T09:50:33,764 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:33,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T09:50:33,765 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,768 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,769 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,775 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,776 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,779 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T09:50:33,783 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-08T09:50:33,787 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T09:50:33,788 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62330888, jitterRate=-0.07119739055633545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T09:50:33,796 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731059433703Initializing all the Stores at 1731059433704 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059433705 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731059433705Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731059433705Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731059433705Cleaning up temporary data from old regions at 1731059433776 (+71 ms)Region opened successfully at 1731059433795 (+19 ms)
2024-11-08T09:50:33,797 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-08T09:50:33,829 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2404d07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cf4d76213909/172.17.0.2:0
2024-11-08T09:50:33,858 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-08T09:50:33,868 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-08T09:50:33,868 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-08T09:50:33,870 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-08T09:50:33,871 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-08T09:50:33,876 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-08T09:50:33,876 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-08T09:50:33,905 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-08T09:50:33,914 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-08T09:50:33,956 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-08T09:50:33,959 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-08T09:50:33,962 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-08T09:50:33,971 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-08T09:50:33,974 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-08T09:50:33,978 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-08T09:50:33,987 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-08T09:50:33,990 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-08T09:50:34,002 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-08T09:50:34,020 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-08T09:50:34,029 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-08T09:50:34,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:34,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:34,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:34,045 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:34,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,045 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,048 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cf4d76213909,45843,1731059432888, sessionid=0x10119dab9480000, setting cluster-up flag (Was=false)
2024-11-08T09:50:34,076 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,108 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-08T09:50:34,113 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cf4d76213909,45843,1731059432888
2024-11-08T09:50:34,139 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,171 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-08T09:50:34,173 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cf4d76213909,45843,1731059432888
2024-11-08T09:50:34,181 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-08T09:50:34,190 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(746): ClusterId : bf0214c8-0430-400d-8f92-2fc658fea88f
2024-11-08T09:50:34,190 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(746): ClusterId : bf0214c8-0430-400d-8f92-2fc658fea88f
2024-11-08T09:50:34,191 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(746): ClusterId : bf0214c8-0430-400d-8f92-2fc658fea88f
2024-11-08T09:50:34,193 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-08T09:50:34,193 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-08T09:50:34,193 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-08T09:50:34,204 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-08T09:50:34,204 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-08T09:50:34,204 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-08T09:50:34,204 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-08T09:50:34,204 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-08T09:50:34,204 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-08T09:50:34,214 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-08T09:50:34,214 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-08T09:50:34,214 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-08T09:50:34,215 DEBUG [RS:0;cf4d76213909:40561 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7793d990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cf4d76213909/172.17.0.2:0
2024-11-08T09:50:34,215 DEBUG [RS:1;cf4d76213909:36537 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6128ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cf4d76213909/172.17.0.2:0
2024-11-08T09:50:34,215 DEBUG [RS:2;cf4d76213909:38083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541f2704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cf4d76213909/172.17.0.2:0
2024-11-08T09:50:34,227 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cf4d76213909:40561
2024-11-08T09:50:34,230 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-08T09:50:34,230 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-08T09:50:34,230 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-08T09:50:34,232 DEBUG [RS:1;cf4d76213909:36537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;cf4d76213909:36537
2024-11-08T09:50:34,232 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;cf4d76213909:38083
2024-11-08T09:50:34,232 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(2659): reportForDuty to master=cf4d76213909,45843,1731059432888 with port=40561, startcode=1731059433238
2024-11-08T09:50:34,233 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-08T09:50:34,233 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-08T09:50:34,233 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-08T09:50:34,233 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-08T09:50:34,233 DEBUG [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-08T09:50:34,233 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-08T09:50:34,234 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(2659): reportForDuty to master=cf4d76213909,45843,1731059432888 with port=38083, startcode=1731059433356
2024-11-08T09:50:34,234 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(2659): reportForDuty to master=cf4d76213909,45843,1731059432888 with port=36537, startcode=1731059433286
2024-11-08T09:50:34,244 DEBUG [RS:2;cf4d76213909:38083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-08T09:50:34,244 DEBUG [RS:0;cf4d76213909:40561 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-08T09:50:34,244 DEBUG [RS:1;cf4d76213909:36537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-08T09:50:34,257 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-08T09:50:34,268 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-08T09:50:34,275 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-08T09:50:34,281 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cf4d76213909,45843,1731059432888
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-08T09:50:34,289 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5
2024-11-08T09:50:34,289 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5
2024-11-08T09:50:34,290 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5
2024-11-08T09:50:34,290 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5
2024-11-08T09:50:34,290 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cf4d76213909:0, corePoolSize=10, maxPoolSize=10
2024-11-08T09:50:34,290 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,290 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cf4d76213909:0, corePoolSize=2, maxPoolSize=2
2024-11-08T09:50:34,290 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,296 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731059464296
2024-11-08T09:50:34,298 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-08T09:50:34,298 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T09:50:34,298 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-08T09:50:34,299 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-08T09:50:34,303 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-08T09:50:34,304 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-08T09:50:34,304 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-08T09:50:34,304 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-08T09:50:34,306 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:34,306 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-08T09:50:34,306 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,312 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-08T09:50:34,315 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-08T09:50:34,315 INFO  [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55673, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-08T09:50:34,315 INFO  [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-08T09:50:34,315 INFO  [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58001, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-08T09:50:34,316 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-08T09:50:34,320 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-08T09:50:34,321 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-08T09:50:34,323 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.large.0-1731059434322,5,FailOnTimeoutGroup]
2024-11-08T09:50:34,323 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-08T09:50:34,324 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.small.0-1731059434323,5,FailOnTimeoutGroup]
2024-11-08T09:50:34,324 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,324 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-08T09:50:34,326 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,326 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741831_1007 (size=1321)
2024-11-08T09:50:34,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741831_1007 (size=1321)
2024-11-08T09:50:34,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741831_1007 (size=1321)
2024-11-08T09:50:34,330 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-08T09:50:34,331 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cf4d76213909,36537,1731059433286
2024-11-08T09:50:34,331 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3
2024-11-08T09:50:34,333 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] master.ServerManager(517): Registering regionserver=cf4d76213909,36537,1731059433286
2024-11-08T09:50:34,341 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cf4d76213909,40561,1731059433238
2024-11-08T09:50:34,341 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] master.ServerManager(517): Registering regionserver=cf4d76213909,40561,1731059433238
2024-11-08T09:50:34,342 DEBUG [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3
2024-11-08T09:50:34,342 DEBUG [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41199
2024-11-08T09:50:34,342 DEBUG [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-08T09:50:34,346 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3
2024-11-08T09:50:34,346 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41199
2024-11-08T09:50:34,346 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-08T09:50:34,352 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-11-08T09:50:34,352 WARN  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-11-08T09:50:34,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T09:50:34,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741832_1008 (size=32)
2024-11-08T09:50:34,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741832_1008 (size=32)
2024-11-08T09:50:34,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741832_1008 (size=32)
2024-11-08T09:50:34,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T09:50:34,363 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-08T09:50:34,366 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-08T09:50:34,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:34,367 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:34,367 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-08T09:50:34,370 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-08T09:50:34,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:34,371 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:34,371 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-08T09:50:34,373 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-08T09:50:34,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:34,374 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:34,375 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-08T09:50:34,377 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-08T09:50:34,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:34,378 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:34,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-08T09:50:34,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740
2024-11-08T09:50:34,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740
2024-11-08T09:50:34,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-08T09:50:34,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-08T09:50:34,383 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T09:50:34,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-08T09:50:34,389 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T09:50:34,389 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61541129, jitterRate=-0.08296571671962738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T09:50:34,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731059434361Initializing all the Stores at 1731059434363 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059434363Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059434363Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731059434363Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059434363Cleaning up temporary data from old regions at 1731059434382 (+19 ms)Region opened successfully at 1731059434393 (+11 ms)
2024-11-08T09:50:34,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-08T09:50:34,393 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-08T09:50:34,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-08T09:50:34,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-08T09:50:34,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-08T09:50:34,395 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-08T09:50:34,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731059434393Disabling compacts and flushes for region at 1731059434393Disabling writes for close at 1731059434394 (+1 ms)Writing region close event to WAL at 1731059434395 (+1 ms)Closed at 1731059434395
2024-11-08T09:50:34,396 DEBUG [RS:1;cf4d76213909:36537 {}] zookeeper.ZKUtil(111): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cf4d76213909,36537,1731059433286
2024-11-08T09:50:34,396 WARN  [RS:1;cf4d76213909:36537 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T09:50:34,396 INFO  [RS:1;cf4d76213909:36537 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-08T09:50:34,396 DEBUG [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,36537,1731059433286
2024-11-08T09:50:34,397 DEBUG [RS:0;cf4d76213909:40561 {}] zookeeper.ZKUtil(111): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cf4d76213909,40561,1731059433238
2024-11-08T09:50:34,397 WARN  [RS:0;cf4d76213909:40561 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T09:50:34,397 INFO  [RS:0;cf4d76213909:40561 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-08T09:50:34,397 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,40561,1731059433238
2024-11-08T09:50:34,398 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cf4d76213909,36537,1731059433286]
2024-11-08T09:50:34,398 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cf4d76213909,40561,1731059433238]
2024-11-08T09:50:34,398 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T09:50:34,399 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-08T09:50:34,405 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-08T09:50:34,414 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-08T09:50:34,417 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-08T09:50:34,421 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-08T09:50:34,421 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-08T09:50:34,437 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-08T09:50:34,437 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-08T09:50:34,442 INFO  [RS:1;cf4d76213909:36537 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-08T09:50:34,442 INFO  [RS:0;cf4d76213909:40561 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-08T09:50:34,442 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,442 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,443 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-08T09:50:34,443 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-08T09:50:34,449 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-08T09:50:34,449 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-08T09:50:34,450 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,450 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,450 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,450 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,450 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,450 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cf4d76213909:0, corePoolSize=2, maxPoolSize=2
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cf4d76213909:0, corePoolSize=2, maxPoolSize=2
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,451 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,452 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3
2024-11-08T09:50:34,452 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,452 DEBUG [RS:0;cf4d76213909:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3
2024-11-08T09:50:34,452 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,452 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3
2024-11-08T09:50:34,452 DEBUG [RS:1;cf4d76213909:36537 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3
2024-11-08T09:50:34,453 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(2659): reportForDuty to master=cf4d76213909,45843,1731059432888 with port=38083, startcode=1731059433356
2024-11-08T09:50:34,453 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,40561,1731059433238-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T09:50:34,453 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,36537,1731059433286-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T09:50:34,454 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,455 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45843 {}] master.ServerManager(517): Registering regionserver=cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,457 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3
2024-11-08T09:50:34,458 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41199
2024-11-08T09:50:34,458 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-08T09:50:34,473 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-08T09:50:34,473 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-08T09:50:34,475 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,36537,1731059433286-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,475 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,40561,1731059433238-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,475 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,475 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,475 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.Replication(171): cf4d76213909,40561,1731059433238 started
2024-11-08T09:50:34,475 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.Replication(171): cf4d76213909,36537,1731059433286 started
2024-11-08T09:50:34,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T09:50:34,477 DEBUG [RS:2;cf4d76213909:38083 {}] zookeeper.ZKUtil(111): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,477 WARN  [RS:2;cf4d76213909:38083 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-08T09:50:34,477 INFO  [RS:2;cf4d76213909:38083 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-08T09:50:34,477 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cf4d76213909,38083,1731059433356]
2024-11-08T09:50:34,477 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,484 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-08T09:50:34,488 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-08T09:50:34,491 INFO  [RS:2;cf4d76213909:38083 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-08T09:50:34,491 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,491 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-08T09:50:34,493 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-08T09:50:34,493 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cf4d76213909:0, corePoolSize=2, maxPoolSize=2
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,493 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,494 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,494 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,494 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,494 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1
2024-11-08T09:50:34,494 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3
2024-11-08T09:50:34,494 DEBUG [RS:2;cf4d76213909:38083 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3
2024-11-08T09:50:34,496 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,496 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,496 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,496 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,496 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(1482): Serving as cf4d76213909,36537,1731059433286, RpcServer on cf4d76213909/172.17.0.2:36537, sessionid=0x10119dab9480002
2024-11-08T09:50:34,496 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1482): Serving as cf4d76213909,40561,1731059433238, RpcServer on cf4d76213909/172.17.0.2:40561, sessionid=0x10119dab9480001
2024-11-08T09:50:34,496 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,497 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,497 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,497 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,38083,1731059433356-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T09:50:34,497 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-08T09:50:34,497 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-08T09:50:34,497 DEBUG [RS:1;cf4d76213909:36537 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cf4d76213909,36537,1731059433286
2024-11-08T09:50:34,497 DEBUG [RS:0;cf4d76213909:40561 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cf4d76213909,40561,1731059433238
2024-11-08T09:50:34,497 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,36537,1731059433286'
2024-11-08T09:50:34,497 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,40561,1731059433238'
2024-11-08T09:50:34,497 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-08T09:50:34,497 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-08T09:50:34,498 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-08T09:50:34,498 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-08T09:50:34,499 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-08T09:50:34,499 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-08T09:50:34,499 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-08T09:50:34,499 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-08T09:50:34,499 DEBUG [RS:0;cf4d76213909:40561 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cf4d76213909,40561,1731059433238
2024-11-08T09:50:34,499 DEBUG [RS:1;cf4d76213909:36537 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cf4d76213909,36537,1731059433286
2024-11-08T09:50:34,499 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,40561,1731059433238'
2024-11-08T09:50:34,499 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,36537,1731059433286'
2024-11-08T09:50:34,499 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-08T09:50:34,499 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-08T09:50:34,500 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-08T09:50:34,500 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-08T09:50:34,500 DEBUG [RS:0;cf4d76213909:40561 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-08T09:50:34,500 DEBUG [RS:1;cf4d76213909:36537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-08T09:50:34,500 INFO  [RS:1;cf4d76213909:36537 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-08T09:50:34,500 INFO  [RS:0;cf4d76213909:40561 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-08T09:50:34,500 INFO  [RS:1;cf4d76213909:36537 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-08T09:50:34,500 INFO  [RS:0;cf4d76213909:40561 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-08T09:50:34,513 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-08T09:50:34,513 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,38083,1731059433356-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,514 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,514 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.Replication(171): cf4d76213909,38083,1731059433356 started
2024-11-08T09:50:34,529 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:34,529 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1482): Serving as cf4d76213909,38083,1731059433356, RpcServer on cf4d76213909/172.17.0.2:38083, sessionid=0x10119dab9480003
2024-11-08T09:50:34,529 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-08T09:50:34,529 DEBUG [RS:2;cf4d76213909:38083 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,529 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,38083,1731059433356'
2024-11-08T09:50:34,529 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-08T09:50:34,530 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-08T09:50:34,531 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-08T09:50:34,531 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-08T09:50:34,531 DEBUG [RS:2;cf4d76213909:38083 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,531 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,38083,1731059433356'
2024-11-08T09:50:34,531 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-08T09:50:34,531 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-08T09:50:34,532 DEBUG [RS:2;cf4d76213909:38083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-08T09:50:34,532 INFO  [RS:2;cf4d76213909:38083 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-08T09:50:34,532 INFO  [RS:2;cf4d76213909:38083 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-08T09:50:34,568 WARN  [cf4d76213909:45843 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-08T09:50:34,608 INFO  [RS:1;cf4d76213909:36537 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C36537%2C1731059433286, suffix=, logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,36537,1731059433286, archiveDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs, maxLogs=32
2024-11-08T09:50:34,608 INFO  [RS:0;cf4d76213909:40561 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C40561%2C1731059433238, suffix=, logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,40561,1731059433238, archiveDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs, maxLogs=32
2024-11-08T09:50:34,612 INFO  [RS:0;cf4d76213909:40561 {}] monitor.StreamSlowMonitor(122): New stream slow monitor cf4d76213909%2C40561%2C1731059433238.1731059434611
2024-11-08T09:50:34,612 INFO  [RS:1;cf4d76213909:36537 {}] monitor.StreamSlowMonitor(122): New stream slow monitor cf4d76213909%2C36537%2C1731059433286.1731059434611
2024-11-08T09:50:34,625 INFO  [RS:0;cf4d76213909:40561 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,40561,1731059433238/cf4d76213909%2C40561%2C1731059433238.1731059434611
2024-11-08T09:50:34,626 INFO  [RS:1;cf4d76213909:36537 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,36537,1731059433286/cf4d76213909%2C36537%2C1731059433286.1731059434611
2024-11-08T09:50:34,630 DEBUG [RS:0;cf4d76213909:40561 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46277:46277),(127.0.0.1/127.0.0.1:36767:36767),(127.0.0.1/127.0.0.1:44491:44491)]
2024-11-08T09:50:34,630 DEBUG [RS:1;cf4d76213909:36537 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46277:46277),(127.0.0.1/127.0.0.1:44491:44491),(127.0.0.1/127.0.0.1:36767:36767)]
2024-11-08T09:50:34,635 INFO  [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C38083%2C1731059433356, suffix=, logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,38083,1731059433356, archiveDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs, maxLogs=32
2024-11-08T09:50:34,637 INFO  [RS:2;cf4d76213909:38083 {}] monitor.StreamSlowMonitor(122): New stream slow monitor cf4d76213909%2C38083%2C1731059433356.1731059434637
2024-11-08T09:50:34,651 INFO  [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,38083,1731059433356/cf4d76213909%2C38083%2C1731059433356.1731059434637
2024-11-08T09:50:34,652 DEBUG [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36767:36767),(127.0.0.1/127.0.0.1:44491:44491),(127.0.0.1/127.0.0.1:46277:46277)]
2024-11-08T09:50:34,825 DEBUG [cf4d76213909:45843 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3
2024-11-08T09:50:34,834 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(204): Hosts are {cf4d76213909=0} racks are {/default-rack=0}
2024-11-08T09:50:34,840 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-08T09:50:34,840 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-08T09:50:34,840 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(303): server 2 has 0 regions
2024-11-08T09:50:34,840 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-08T09:50:34,841 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-08T09:50:34,841 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(310): server 2 is on host 0
2024-11-08T09:50:34,841 INFO  [cf4d76213909:45843 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-08T09:50:34,841 INFO  [cf4d76213909:45843 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-08T09:50:34,841 INFO  [cf4d76213909:45843 {}] balancer.BalancerClusterState(321): server 2 is on rack 0
2024-11-08T09:50:34,841 DEBUG [cf4d76213909:45843 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-08T09:50:34,847 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cf4d76213909,38083,1731059433356
2024-11-08T09:50:34,853 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cf4d76213909,38083,1731059433356, state=OPENING
2024-11-08T09:50:34,865 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-08T09:50:34,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,876 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:34,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:34,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:34,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:34,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:34,879 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-08T09:50:34,881 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cf4d76213909,38083,1731059433356}]
2024-11-08T09:50:35,054 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-08T09:50:35,056 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57373, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-08T09:50:35,067 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-08T09:50:35,068 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-08T09:50:35,071 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C38083%2C1731059433356.meta, suffix=.meta, logDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,38083,1731059433356, archiveDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs, maxLogs=32
2024-11-08T09:50:35,074 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor cf4d76213909%2C38083%2C1731059433356.meta.1731059435074.meta
2024-11-08T09:50:35,088 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/WALs/cf4d76213909,38083,1731059433356/cf4d76213909%2C38083%2C1731059433356.meta.1731059435074.meta
2024-11-08T09:50:35,089 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46277:46277),(127.0.0.1/127.0.0.1:44491:44491),(127.0.0.1/127.0.0.1:36767:36767)]
2024-11-08T09:50:35,090 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-08T09:50:35,092 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-08T09:50:35,095 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-08T09:50:35,100 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-08T09:50:35,104 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-08T09:50:35,105 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T09:50:35,105 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-08T09:50:35,105 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-08T09:50:35,108 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-08T09:50:35,110 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-08T09:50:35,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:35,110 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:35,111 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-08T09:50:35,112 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-08T09:50:35,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:35,113 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:35,113 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-08T09:50:35,114 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-08T09:50:35,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:35,115 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:35,115 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-08T09:50:35,116 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-08T09:50:35,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:35,117 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-08T09:50:35,117 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-08T09:50:35,119 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740
2024-11-08T09:50:35,121 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740
2024-11-08T09:50:35,123 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-08T09:50:35,123 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-08T09:50:35,124 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-08T09:50:35,126 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-08T09:50:35,127 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63038050, jitterRate=-0.06065985560417175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-08T09:50:35,128 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-08T09:50:35,129 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731059435105Writing region info on filesystem at 1731059435106 (+1 ms)Initializing all the Stores at 1731059435108 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059435108Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059435108Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731059435108Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731059435108Cleaning up temporary data from old regions at 1731059435123 (+15 ms)Running coprocessor post-open hooks at 1731059435128 (+5 ms)Region opened successfully at 1731059435129 (+1 ms)
2024-11-08T09:50:35,135 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731059435045
2024-11-08T09:50:35,147 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-08T09:50:35,147 INFO  [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-08T09:50:35,149 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,38083,1731059433356
2024-11-08T09:50:35,152 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cf4d76213909,38083,1731059433356, state=OPEN
2024-11-08T09:50:35,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T09:50:35,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T09:50:35,208 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T09:50:35,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-08T09:50:35,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:35,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:35,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:35,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-08T09:50:35,209 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cf4d76213909,38083,1731059433356
2024-11-08T09:50:35,215 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-08T09:50:35,215 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cf4d76213909,38083,1731059433356 in 329 msec
2024-11-08T09:50:35,222 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-08T09:50:35,222 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 811 msec
2024-11-08T09:50:35,224 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-08T09:50:35,224 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-08T09:50:35,240 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-08T09:50:35,241 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,38083,1731059433356, seqNum=-1]
2024-11-08T09:50:35,259 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T09:50:35,261 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37115, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T09:50:35,281 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0680 sec
2024-11-08T09:50:35,281 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731059435281, completionTime=-1
2024-11-08T09:50:35,283 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running
2024-11-08T09:50:35,283 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-08T09:50:35,306 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3
2024-11-08T09:50:35,306 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731059495306
2024-11-08T09:50:35,306 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731059555306
2024-11-08T09:50:35,306 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec
2024-11-08T09:50:35,308 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache
2024-11-08T09:50:35,314 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,45843,1731059432888-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,315 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,45843,1731059432888-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,315 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,45843,1731059432888-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,316 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cf4d76213909:45843, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,336 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,336 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,343 DEBUG [master/cf4d76213909:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-08T09:50:35,365 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.929sec
2024-11-08T09:50:35,366 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-08T09:50:35,367 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-08T09:50:35,368 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-08T09:50:35,368 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-08T09:50:35,368 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-08T09:50:35,369 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,45843,1731059432888-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-08T09:50:35,369 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,45843,1731059432888-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-08T09:50:35,378 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-08T09:50:35,379 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-08T09:50:35,380 INFO  [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,45843,1731059432888-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-08T09:50:35,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7bc1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T09:50:35,401 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-08T09:50:35,401 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-08T09:50:35,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,45843,-1 for getting cluster id
2024-11-08T09:50:35,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-08T09:50:35,418 DEBUG [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bf0214c8-0430-400d-8f92-2fc658fea88f'
2024-11-08T09:50:35,420 DEBUG [RPCClient-NioEventLoopGroup-10-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-08T09:50:35,421 DEBUG [RPCClient-NioEventLoopGroup-10-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bf0214c8-0430-400d-8f92-2fc658fea88f"

2024-11-08T09:50:35,421 DEBUG [RPCClient-NioEventLoopGroup-10-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf5eb31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T09:50:35,421 DEBUG [RPCClient-NioEventLoopGroup-10-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,45843,-1]
2024-11-08T09:50:35,424 DEBUG [RPCClient-NioEventLoopGroup-10-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-08T09:50:35,425 DEBUG [RPCClient-NioEventLoopGroup-10-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:35,426 INFO  [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-08T09:50:35,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75325a43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-08T09:50:35,430 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-08T09:50:35,437 DEBUG [RPCClient-NioEventLoopGroup-10-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,38083,1731059433356, seqNum=-1]
2024-11-08T09:50:35,438 DEBUG [RPCClient-NioEventLoopGroup-10-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T09:50:35,440 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T09:50:35,458 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cf4d76213909,45843,1731059432888
2024-11-08T09:50:35,461 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-08T09:50:35,466 DEBUG [RPCClient-NioEventLoopGroup-10-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,45843,1731059432888
2024-11-08T09:50:35,468 DEBUG [RPCClient-NioEventLoopGroup-10-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4bfdc29d
2024-11-08T09:50:35,469 DEBUG [RPCClient-NioEventLoopGroup-10-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-08T09:50:35,472 INFO  [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-08T09:50:35,477 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-08T09:50:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC
2024-11-08T09:50:35,488 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_PRE_OPERATION
2024-11-08T09:50:35,490 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestHBaseWalOnEC"
 procId is: 4
2024-11-08T09:50:35,490 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:35,493 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-08T09:50:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-08T09:50:35,499 WARN  [IPC Server handler 0 on default port 41199 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:35,499 WARN  [IPC Server handler 0 on default port 41199 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-08T09:50:35,500 WARN  [IPC Server handler 0 on default port 41199 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-08T09:50:35,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741837_1013 (size=392)
2024-11-08T09:50:35,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741837_1013 (size=392)
2024-11-08T09:50:35,510 INFO  [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cc10ad126524541f1652b133eadf2e75, NAME => 'TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3
2024-11-08T09:50:35,515 WARN  [IPC Server handler 1 on default port 41199 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-08T09:50:35,515 WARN  [IPC Server handler 1 on default port 41199 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-08T09:50:35,515 WARN  [IPC Server handler 1 on default port 41199 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-08T09:50:35,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741838_1014 (size=51)
2024-11-08T09:50:35,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741838_1014 (size=51)
2024-11-08T09:50:35,522 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T09:50:35,523 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1722): Closing cc10ad126524541f1652b133eadf2e75, disabling compactions & flushes
2024-11-08T09:50:35,523 INFO  [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,523 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,523 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75. after waiting 0 ms
2024-11-08T09:50:35,523 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,523 INFO  [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,523 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1676): Region close journal for cc10ad126524541f1652b133eadf2e75:
Waiting for close lock at 1731059435523Disabling compacts and flushes for region at 1731059435523Disabling writes for close at 1731059435523Writing region close event to WAL at 1731059435523Closed at 1731059435523
2024-11-08T09:50:35,526 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ADD_TO_META
2024-11-08T09:50:35,532 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731059435526"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731059435526"}]},"ts":"1731059435526"}
2024-11-08T09:50:35,537 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-08T09:50:35,540 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-08T09:50:35,543 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731059435540"}]},"ts":"1731059435540"}
2024-11-08T09:50:35,547 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLING in hbase:meta
2024-11-08T09:50:35,548 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {cf4d76213909=0} racks are {/default-rack=0}
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0
2024-11-08T09:50:35,549 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-08T09:50:35,549 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-08T09:50:35,549 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0
2024-11-08T09:50:35,549 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-08T09:50:35,551 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=cc10ad126524541f1652b133eadf2e75, ASSIGN}]
2024-11-08T09:50:35,554 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=cc10ad126524541f1652b133eadf2e75, ASSIGN
2024-11-08T09:50:35,557 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=cc10ad126524541f1652b133eadf2e75, ASSIGN; state=OFFLINE, location=cf4d76213909,40561,1731059433238; forceNewPlan=false, retain=false
2024-11-08T09:50:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-08T09:50:35,709 INFO  [cf4d76213909:45843 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-08T09:50:35,710 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc10ad126524541f1652b133eadf2e75, regionState=OPENING, regionLocation=cf4d76213909,40561,1731059433238
2024-11-08T09:50:35,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=cc10ad126524541f1652b133eadf2e75, ASSIGN because future has completed
2024-11-08T09:50:35,717 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc10ad126524541f1652b133eadf2e75, server=cf4d76213909,40561,1731059433238}]
2024-11-08T09:50:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-08T09:50:35,872 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-08T09:50:35,875 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-08T09:50:35,882 INFO  [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,882 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cc10ad126524541f1652b133eadf2e75, NAME => 'TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.', STARTKEY => '', ENDKEY => ''}
2024-11-08T09:50:35,883 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestHBaseWalOnEC cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,883 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-08T09:50:35,883 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,883 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,886 INFO  [StoreOpener-cc10ad126524541f1652b133eadf2e75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region cc10ad126524541f1652b133eadf2e75 
2024-11-08T09:50:35,889 INFO  [StoreOpener-cc10ad126524541f1652b133eadf2e75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc10ad126524541f1652b133eadf2e75 columnFamilyName cf
2024-11-08T09:50:35,890 DEBUG [StoreOpener-cc10ad126524541f1652b133eadf2e75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-08T09:50:35,891 INFO  [StoreOpener-cc10ad126524541f1652b133eadf2e75-1 {}] regionserver.HStore(327): Store=cc10ad126524541f1652b133eadf2e75/cf,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-08T09:50:35,891 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,893 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,894 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,895 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,895 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,899 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,903 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-08T09:50:35,903 INFO  [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cc10ad126524541f1652b133eadf2e75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63803133, jitterRate=-0.04925923049449921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-08T09:50:35,904 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:35,905 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cc10ad126524541f1652b133eadf2e75:
Running coprocessor pre-open hook at 1731059435883Writing region info on filesystem at 1731059435883Initializing all the Stores at 1731059435886 (+3 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731059435886Cleaning up temporary data from old regions at 1731059435895 (+9 ms)Running coprocessor post-open hooks at 1731059435904 (+9 ms)Region opened successfully at 1731059435905 (+1 ms)
2024-11-08T09:50:35,907 INFO  [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75., pid=6, masterSystemTime=1731059435872
2024-11-08T09:50:35,910 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,911 INFO  [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:35,912 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cc10ad126524541f1652b133eadf2e75, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,40561,1731059433238
2024-11-08T09:50:35,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc10ad126524541f1652b133eadf2e75, server=cf4d76213909,40561,1731059433238 because future has completed
2024-11-08T09:50:35,923 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-08T09:50:35,925 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cc10ad126524541f1652b133eadf2e75, server=cf4d76213909,40561,1731059433238 in 201 msec
2024-11-08T09:50:35,929 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-08T09:50:35,929 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=cc10ad126524541f1652b133eadf2e75, ASSIGN in 372 msec
2024-11-08T09:50:35,930 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-08T09:50:35,930 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731059435930"}]},"ts":"1731059435930"}
2024-11-08T09:50:35,934 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLED in hbase:meta
2024-11-08T09:50:35,936 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_POST_OPERATION
2024-11-08T09:50:35,940 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC in 456 msec
2024-11-08T09:50:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-08T09:50:36,125 INFO  [RPCClient-NioEventLoopGroup-10-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestHBaseWalOnEC completed
2024-11-08T09:50:36,125 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestHBaseWalOnEC get assigned. Timeout = 60000ms
2024-11-08T09:50:36,127 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1])
2024-11-08T09:50:36,134 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestHBaseWalOnEC assigned to meta. Checking AM states.
2024-11-08T09:50:36,135 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1])
2024-11-08T09:50:36,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestHBaseWalOnEC assigned.
2024-11-08T09:50:36,145 DEBUG [RPCClient-NioEventLoopGroup-10-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestHBaseWalOnEC', row='row', locateType=CURRENT is [region=TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75., hostname=cf4d76213909,40561,1731059433238, seqNum=2]
2024-11-08T09:50:36,147 DEBUG [RPCClient-NioEventLoopGroup-10-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-08T09:50:36,149 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-08T09:50:36,156 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestHBaseWalOnEC
2024-11-08T09:50:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC
2024-11-08T09:50:36,163 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_PREPARE
2024-11-08T09:50:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-08T09:50:36,165 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-08T09:50:36,166 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-08T09:50:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-08T09:50:36,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40561 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-08T09:50:36,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,334 INFO  [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cc10ad126524541f1652b133eadf2e75 1/1 column families, dataSize=32 B heapSize=360 B
2024-11-08T09:50:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/.tmp/cf/f1efe046870d434bbb25130d47aa2f84 is 36, key is row/cf:cq/1731059436150/Put/seqid=0
2024-11-08T09:50:36,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741839_1015 (size=4787)
2024-11-08T09:50:36,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741839_1015 (size=4787)
2024-11-08T09:50:36,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741839_1015 (size=4787)
2024-11-08T09:50:36,395 INFO  [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/.tmp/cf/f1efe046870d434bbb25130d47aa2f84
2024-11-08T09:50:36,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/.tmp/cf/f1efe046870d434bbb25130d47aa2f84 as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/cf/f1efe046870d434bbb25130d47aa2f84
2024-11-08T09:50:36,453 INFO  [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/cf/f1efe046870d434bbb25130d47aa2f84, entries=1, sequenceid=5, filesize=4.7 K
2024-11-08T09:50:36,461 INFO  [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~32 B/32, heapSize ~344 B/344, currentSize=0 B/0 for cc10ad126524541f1652b133eadf2e75 in 127ms, sequenceid=5, compaction requested=false
2024-11-08T09:50:36,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestHBaseWalOnEC'
2024-11-08T09:50:36,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cc10ad126524541f1652b133eadf2e75:

2024-11-08T09:50:36,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-08T09:50:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-08T09:50:36,475 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-08T09:50:36,475 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 304 msec
2024-11-08T09:50:36,479 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC in 319 msec
2024-11-08T09:50:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45843 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-08T09:50:36,485 INFO  [RPCClient-NioEventLoopGroup-10-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestHBaseWalOnEC completed
2024-11-08T09:50:36,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-08T09:50:36,497 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-08T09:50:36,498 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
    at org.junit.runners.Suite.runChild(Suite.java:128)
    at org.junit.runners.Suite.runChild(Suite.java:27)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T09:50:36,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,499 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-08T09:50:36,499 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-08T09:50:36,499 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1231802991, stopped=false
2024-11-08T09:50:36,499 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cf4d76213909,45843,1731059432888
2024-11-08T09:50:36,566 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-08T09:50:36,567 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:36,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:36,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:36,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:36,567 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T09:50:36,568 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-08T09:50:36,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
    at org.junit.runners.Suite.runChild(Suite.java:128)
    at org.junit.runners.Suite.runChild(Suite.java:27)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T09:50:36,568 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:36,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,568 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:36,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:36,569 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,40561,1731059433238' *****
2024-11-08T09:50:36,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-08T09:50:36,569 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T09:50:36,569 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,36537,1731059433286' *****
2024-11-08T09:50:36,570 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T09:50:36,570 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,38083,1731059433356' *****
2024-11-08T09:50:36,570 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-08T09:50:36,570 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-08T09:50:36,571 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-08T09:50:36,571 INFO  [RS:0;cf4d76213909:40561 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-08T09:50:36,571 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-08T09:50:36,571 INFO  [RS:1;cf4d76213909:36537 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-08T09:50:36,571 INFO  [RS:0;cf4d76213909:40561 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-08T09:50:36,571 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-08T09:50:36,571 INFO  [RS:1;cf4d76213909:36537 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-08T09:50:36,571 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(959): stopping server cf4d76213909,36537,1731059433286
2024-11-08T09:50:36,571 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(3091): Received CLOSE for cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:36,572 INFO  [RS:1;cf4d76213909:36537 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T09:50:36,572 INFO  [RS:1;cf4d76213909:36537 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;cf4d76213909:36537.
2024-11-08T09:50:36,572 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-08T09:50:36,572 DEBUG [RS:1;cf4d76213909:36537 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T09:50:36,572 INFO  [RS:2;cf4d76213909:38083 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-08T09:50:36,572 DEBUG [RS:1;cf4d76213909:36537 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,572 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-08T09:50:36,572 INFO  [RS:2;cf4d76213909:38083 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-08T09:50:36,572 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(959): stopping server cf4d76213909,38083,1731059433356
2024-11-08T09:50:36,572 INFO  [RS:2;cf4d76213909:38083 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T09:50:36,572 INFO  [RS:2;cf4d76213909:38083 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;cf4d76213909:38083.
2024-11-08T09:50:36,572 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(976): stopping server cf4d76213909,36537,1731059433286; all regions closed.
2024-11-08T09:50:36,572 DEBUG [RS:2;cf4d76213909:38083 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T09:50:36,573 DEBUG [RS:2;cf4d76213909:38083 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,573 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-08T09:50:36,573 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-08T09:50:36,573 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-08T09:50:36,573 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-08T09:50:36,574 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(959): stopping server cf4d76213909,40561,1731059433238
2024-11-08T09:50:36,574 INFO  [RS:0;cf4d76213909:40561 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T09:50:36,574 INFO  [RS:0;cf4d76213909:40561 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cf4d76213909:40561.
2024-11-08T09:50:36,574 DEBUG [RS:0;cf4d76213909:40561 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-08T09:50:36,574 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-08T09:50:36,574 DEBUG [RS:0;cf4d76213909:40561 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,574 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-08T09:50:36,574 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-08T09:50:36,574 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cc10ad126524541f1652b133eadf2e75, disabling compactions & flushes
2024-11-08T09:50:36,574 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1325): Online Regions={cc10ad126524541f1652b133eadf2e75=TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.}
2024-11-08T09:50:36,574 INFO  [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,574 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-08T09:50:36,575 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,575 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-08T09:50:36,575 DEBUG [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1351): Waiting on cc10ad126524541f1652b133eadf2e75
2024-11-08T09:50:36,575 DEBUG [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-08T09:50:36,575 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75. after waiting 0 ms
2024-11-08T09:50:36,575 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-08T09:50:36,575 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,575 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-08T09:50:36,575 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-08T09:50:36,575 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,575 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB
2024-11-08T09:50:36,575 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,575 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,575 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,576 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,585 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/default/TestHBaseWalOnEC/cc10ad126524541f1652b133eadf2e75/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1
2024-11-08T09:50:36,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741834_1010 (size=93)
2024-11-08T09:50:36,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741834_1010 (size=93)
2024-11-08T09:50:36,588 INFO  [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,588 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cc10ad126524541f1652b133eadf2e75:
Waiting for close lock at 1731059436574Running coprocessor pre-close hooks at 1731059436574Disabling compacts and flushes for region at 1731059436574Disabling writes for close at 1731059436575 (+1 ms)Writing region close event to WAL at 1731059436576 (+1 ms)Running coprocessor post-close hooks at 1731059436586 (+10 ms)Closed at 1731059436588 (+2 ms)
2024-11-08T09:50:36,589 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75.
2024-11-08T09:50:36,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741834_1010 (size=93)
2024-11-08T09:50:36,596 DEBUG [RS:1;cf4d76213909:36537 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs
2024-11-08T09:50:36,596 INFO  [RS:1;cf4d76213909:36537 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog cf4d76213909%2C36537%2C1731059433286:(num 1731059434611)
2024-11-08T09:50:36,596 DEBUG [RS:1;cf4d76213909:36537 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] hbase.ChoreService(370): Chore service for: regionserver/cf4d76213909:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-08T09:50:36,597 INFO  [regionserver/cf4d76213909:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-08T09:50:36,597 INFO  [RS:1;cf4d76213909:36537 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T09:50:36,598 INFO  [RS:1;cf4d76213909:36537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36537
2024-11-08T09:50:36,599 INFO  [regionserver/cf4d76213909:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T09:50:36,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cf4d76213909,36537,1731059433286
2024-11-08T09:50:36,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T09:50:36,608 INFO  [RS:1;cf4d76213909:36537 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T09:50:36,615 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/info/23fd30445acc42c1bb0ab320822b275c is 153, key is TestHBaseWalOnEC,,1731059435473.cc10ad126524541f1652b133eadf2e75./info:regioninfo/1731059435912/Put/seqid=0
2024-11-08T09:50:36,619 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cf4d76213909,36537,1731059433286]
2024-11-08T09:50:36,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741840_1016 (size=6637)
2024-11-08T09:50:36,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741840_1016 (size=6637)
2024-11-08T09:50:36,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741840_1016 (size=6637)
2024-11-08T09:50:36,625 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/info/23fd30445acc42c1bb0ab320822b275c
2024-11-08T09:50:36,629 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cf4d76213909,36537,1731059433286 already deleted, retry=false
2024-11-08T09:50:36,629 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cf4d76213909,36537,1731059433286 expired; onlineServers=2
2024-11-08T09:50:36,656 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/ns/f03c797da40e4475a479e2c9a9d6eb3e is 43, key is default/ns:d/1731059435265/Put/seqid=0
2024-11-08T09:50:36,656 INFO  [regionserver/cf4d76213909:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T09:50:36,656 INFO  [regionserver/cf4d76213909:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T09:50:36,665 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741841_1017 (size=5153)
2024-11-08T09:50:36,665 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741841_1017 (size=5153)
2024-11-08T09:50:36,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741841_1017 (size=5153)
2024-11-08T09:50:36,666 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/ns/f03c797da40e4475a479e2c9a9d6eb3e
2024-11-08T09:50:36,691 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/table/c6912e4f22fa47f488f34ec249c5c1ff is 52, key is TestHBaseWalOnEC/table:state/1731059435930/Put/seqid=0
2024-11-08T09:50:36,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741842_1018 (size=5249)
2024-11-08T09:50:36,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741842_1018 (size=5249)
2024-11-08T09:50:36,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741842_1018 (size=5249)
2024-11-08T09:50:36,700 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=96 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/table/c6912e4f22fa47f488f34ec249c5c1ff
2024-11-08T09:50:36,709 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/info/23fd30445acc42c1bb0ab320822b275c as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/info/23fd30445acc42c1bb0ab320822b275c
2024-11-08T09:50:36,719 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/info/23fd30445acc42c1bb0ab320822b275c, entries=10, sequenceid=11, filesize=6.5 K
2024-11-08T09:50:36,719 INFO  [RS:1;cf4d76213909:36537 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T09:50:36,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:36,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36537-0x10119dab9480002, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:36,719 INFO  [RS:1;cf4d76213909:36537 {}] regionserver.HRegionServer(1031): Exiting; stopping=cf4d76213909,36537,1731059433286; zookeeper connection closed.
2024-11-08T09:50:36,719 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@194e5680 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@194e5680
2024-11-08T09:50:36,721 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/ns/f03c797da40e4475a479e2c9a9d6eb3e as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/ns/f03c797da40e4475a479e2c9a9d6eb3e
2024-11-08T09:50:36,731 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/ns/f03c797da40e4475a479e2c9a9d6eb3e, entries=2, sequenceid=11, filesize=5.0 K
2024-11-08T09:50:36,732 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/.tmp/table/c6912e4f22fa47f488f34ec249c5c1ff as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/table/c6912e4f22fa47f488f34ec249c5c1ff
2024-11-08T09:50:36,742 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/table/c6912e4f22fa47f488f34ec249c5c1ff, entries=2, sequenceid=11, filesize=5.1 K
2024-11-08T09:50:36,744 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 168ms, sequenceid=11, compaction requested=false
2024-11-08T09:50:36,744 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-08T09:50:36,750 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-08T09:50:36,751 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-08T09:50:36,751 INFO  [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-08T09:50:36,751 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731059436574Running coprocessor pre-close hooks at 1731059436574Disabling compacts and flushes for region at 1731059436574Disabling writes for close at 1731059436575 (+1 ms)Obtaining lock to block concurrent updates at 1731059436575Preparing flush snapshotting stores in 1588230740 at 1731059436575Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1377, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1731059436576 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731059436577 (+1 ms)Flushing 1588230740/info: creating writer at 1731059436578 (+1 ms)Flushing 1588230740/info: appending metadata at 1731059436608 (+30 ms)Flushing 1588230740/info: closing flushed file at 1731059436608Flushing 1588230740/ns: creating writer at 1731059436637 (+29 ms)Flushing 1588230740/ns: appending metadata at 1731059436654 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731059436654Flushing 1588230740/table: creating writer at 1731059436675 (+21 ms)Flushing 1588230740/table: appending metadata at 1731059436690 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731059436690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5390ca: reopening flushed file at 1731059436708 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205e81db: reopening flushed file at 1731059436719 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3010a7e7: reopening flushed file at 1731059436731 (+12 ms)Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 168ms, sequenceid=11, compaction requested=false at 1731059436744 (+13 ms)Writing region close event to WAL at 1731059436745 (+1 ms)Running coprocessor post-close hooks at 1731059436751 (+6 ms)Closed at 1731059436751
2024-11-08T09:50:36,751 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-08T09:50:36,775 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(976): stopping server cf4d76213909,38083,1731059433356; all regions closed.
2024-11-08T09:50:36,775 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(976): stopping server cf4d76213909,40561,1731059433238; all regions closed.
2024-11-08T09:50:36,775 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,775 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,775 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,776 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741836_1012 (size=2751)
2024-11-08T09:50:36,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741833_1009 (size=1298)
2024-11-08T09:50:36,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741833_1009 (size=1298)
2024-11-08T09:50:36,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741836_1012 (size=2751)
2024-11-08T09:50:36,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741833_1009 (size=1298)
2024-11-08T09:50:36,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741836_1012 (size=2751)
2024-11-08T09:50:36,785 DEBUG [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs
2024-11-08T09:50:36,785 INFO  [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog cf4d76213909%2C38083%2C1731059433356.meta:.meta(num 1731059435074)
2024-11-08T09:50:36,785 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,786 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,786 DEBUG [RS:0;cf4d76213909:40561 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs
2024-11-08T09:50:36,786 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,786 INFO  [RS:0;cf4d76213909:40561 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog cf4d76213909%2C40561%2C1731059433238:(num 1731059434611)
2024-11-08T09:50:36,786 DEBUG [RS:0;cf4d76213909:40561 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,786 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,786 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T09:50:36,786 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:36,786 INFO  [RS:0;cf4d76213909:40561 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T09:50:36,786 INFO  [RS:0;cf4d76213909:40561 {}] hbase.ChoreService(370): Chore service for: regionserver/cf4d76213909:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-08T09:50:36,786 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-08T09:50:36,786 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-08T09:50:36,786 INFO  [regionserver/cf4d76213909:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T09:50:36,787 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-08T09:50:36,787 INFO  [RS:0;cf4d76213909:40561 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T09:50:36,787 INFO  [RS:0;cf4d76213909:40561 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40561
2024-11-08T09:50:36,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741835_1011 (size=93)
2024-11-08T09:50:36,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741835_1011 (size=93)
2024-11-08T09:50:36,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741835_1011 (size=93)
2024-11-08T09:50:36,792 DEBUG [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/oldWALs
2024-11-08T09:50:36,792 INFO  [RS:2;cf4d76213909:38083 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog cf4d76213909%2C38083%2C1731059433356:(num 1731059434637)
2024-11-08T09:50:36,792 DEBUG [RS:2;cf4d76213909:38083 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-08T09:50:36,792 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.LeaseManager(133): Closed leases
2024-11-08T09:50:36,792 INFO  [RS:2;cf4d76213909:38083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T09:50:36,792 INFO  [RS:2;cf4d76213909:38083 {}] hbase.ChoreService(370): Chore service for: regionserver/cf4d76213909:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-08T09:50:36,793 INFO  [RS:2;cf4d76213909:38083 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T09:50:36,793 INFO  [regionserver/cf4d76213909:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T09:50:36,793 INFO  [RS:2;cf4d76213909:38083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38083
2024-11-08T09:50:36,797 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cf4d76213909,40561,1731059433238
2024-11-08T09:50:36,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-08T09:50:36,797 INFO  [RS:0;cf4d76213909:40561 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T09:50:36,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cf4d76213909,38083,1731059433356
2024-11-08T09:50:36,808 INFO  [RS:2;cf4d76213909:38083 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T09:50:36,819 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cf4d76213909,40561,1731059433238]
2024-11-08T09:50:36,840 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cf4d76213909,40561,1731059433238 already deleted, retry=false
2024-11-08T09:50:36,840 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cf4d76213909,40561,1731059433238 expired; onlineServers=1
2024-11-08T09:50:36,840 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cf4d76213909,38083,1731059433356]
2024-11-08T09:50:36,850 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cf4d76213909,38083,1731059433356 already deleted, retry=false
2024-11-08T09:50:36,850 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cf4d76213909,38083,1731059433356 expired; onlineServers=0
2024-11-08T09:50:36,851 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cf4d76213909,45843,1731059432888' *****
2024-11-08T09:50:36,851 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-08T09:50:36,851 INFO  [M:0;cf4d76213909:45843 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-08T09:50:36,851 INFO  [M:0;cf4d76213909:45843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-08T09:50:36,851 DEBUG [M:0;cf4d76213909:45843 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-08T09:50:36,852 DEBUG [M:0;cf4d76213909:45843 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-08T09:50:36,852 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-08T09:50:36,852 DEBUG [master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.small.0-1731059434323 {}] cleaner.HFileCleaner(306): Exit Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.small.0-1731059434323,5,FailOnTimeoutGroup]
2024-11-08T09:50:36,852 DEBUG [master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.large.0-1731059434322 {}] cleaner.HFileCleaner(306): Exit Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.large.0-1731059434322,5,FailOnTimeoutGroup]
2024-11-08T09:50:36,852 INFO  [M:0;cf4d76213909:45843 {}] hbase.ChoreService(370): Chore service for: master/cf4d76213909:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-08T09:50:36,853 INFO  [M:0;cf4d76213909:45843 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-08T09:50:36,853 DEBUG [M:0;cf4d76213909:45843 {}] master.HMaster(1795): Stopping service threads
2024-11-08T09:50:36,853 INFO  [M:0;cf4d76213909:45843 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-08T09:50:36,853 INFO  [M:0;cf4d76213909:45843 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-08T09:50:36,855 INFO  [M:0;cf4d76213909:45843 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-08T09:50:36,855 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-08T09:50:36,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-08T09:50:36,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-08T09:50:36,866 DEBUG [M:0;cf4d76213909:45843 {}] zookeeper.ZKUtil(347): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-08T09:50:36,866 WARN  [M:0;cf4d76213909:45843 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-08T09:50:36,868 INFO  [M:0;cf4d76213909:45843 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/.lastflushedseqids
2024-11-08T09:50:36,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741843_1019 (size=127)
2024-11-08T09:50:36,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741843_1019 (size=127)
2024-11-08T09:50:36,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741843_1019 (size=127)
2024-11-08T09:50:36,887 INFO  [M:0;cf4d76213909:45843 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-08T09:50:36,887 INFO  [M:0;cf4d76213909:45843 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-08T09:50:36,888 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-08T09:50:36,888 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:36,888 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:36,888 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-08T09:50:36,888 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:36,888 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=26.83 KB heapSize=34.12 KB
2024-11-08T09:50:36,907 DEBUG [M:0;cf4d76213909:45843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ca550f0d8ae64b87a73bf3459071a44b is 82, key is hbase:meta,,1/info:regioninfo/1731059435148/Put/seqid=0
2024-11-08T09:50:36,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741844_1020 (size=5672)
2024-11-08T09:50:36,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741844_1020 (size=5672)
2024-11-08T09:50:36,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741844_1020 (size=5672)
2024-11-08T09:50:36,915 INFO  [M:0;cf4d76213909:45843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ca550f0d8ae64b87a73bf3459071a44b
2024-11-08T09:50:36,919 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:36,919 INFO  [RS:0;cf4d76213909:40561 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T09:50:36,919 DEBUG [pool-144-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x10119dab9480001, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:36,919 INFO  [RS:0;cf4d76213909:40561 {}] regionserver.HRegionServer(1031): Exiting; stopping=cf4d76213909,40561,1731059433238; zookeeper connection closed.
2024-11-08T09:50:36,919 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18381e45 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18381e45
2024-11-08T09:50:36,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:36,929 INFO  [RS:2;cf4d76213909:38083 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T09:50:36,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38083-0x10119dab9480003, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:36,929 INFO  [RS:2;cf4d76213909:38083 {}] regionserver.HRegionServer(1031): Exiting; stopping=cf4d76213909,38083,1731059433356; zookeeper connection closed.
2024-11-08T09:50:36,930 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f41e69d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f41e69d
2024-11-08T09:50:36,930 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete
2024-11-08T09:50:36,939 DEBUG [M:0;cf4d76213909:45843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1fe63a037de44b20a2edbdfdf97645bd is 748, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731059435939/Put/seqid=0
2024-11-08T09:50:36,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741845_1021 (size=6439)
2024-11-08T09:50:36,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741845_1021 (size=6439)
2024-11-08T09:50:36,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741845_1021 (size=6439)
2024-11-08T09:50:36,948 INFO  [M:0;cf4d76213909:45843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.14 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1fe63a037de44b20a2edbdfdf97645bd
2024-11-08T09:50:36,969 DEBUG [M:0;cf4d76213909:45843 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/25665c0a1b6343baa4bcecddc9f787af is 69, key is cf4d76213909,36537,1731059433286/rs:state/1731059434335/Put/seqid=0
2024-11-08T09:50:36,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741846_1022 (size=5294)
2024-11-08T09:50:36,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741846_1022 (size=5294)
2024-11-08T09:50:36,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741846_1022 (size=5294)
2024-11-08T09:50:36,977 INFO  [M:0;cf4d76213909:45843 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/25665c0a1b6343baa4bcecddc9f787af
2024-11-08T09:50:36,984 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ca550f0d8ae64b87a73bf3459071a44b as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ca550f0d8ae64b87a73bf3459071a44b
2024-11-08T09:50:36,992 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ca550f0d8ae64b87a73bf3459071a44b, entries=8, sequenceid=72, filesize=5.5 K
2024-11-08T09:50:36,994 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1fe63a037de44b20a2edbdfdf97645bd as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1fe63a037de44b20a2edbdfdf97645bd
2024-11-08T09:50:37,001 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1fe63a037de44b20a2edbdfdf97645bd, entries=8, sequenceid=72, filesize=6.3 K
2024-11-08T09:50:37,002 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/25665c0a1b6343baa4bcecddc9f787af as hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/25665c0a1b6343baa4bcecddc9f787af
2024-11-08T09:50:37,009 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41199/user/jenkins/test-data/600f00cb-fe6e-2329-b756-998a17776df3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/25665c0a1b6343baa4bcecddc9f787af, entries=3, sequenceid=72, filesize=5.2 K
2024-11-08T09:50:37,011 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.83 KB/27471, heapSize ~33.82 KB/34632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 123ms, sequenceid=72, compaction requested=false
2024-11-08T09:50:37,012 INFO  [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-08T09:50:37,013 DEBUG [M:0;cf4d76213909:45843 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731059436888Disabling compacts and flushes for region at 1731059436888Disabling writes for close at 1731059436888Obtaining lock to block concurrent updates at 1731059436888Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731059436888Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=27471, getHeapSize=34872, getOffHeapSize=0, getCellsCount=85 at 1731059436889 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731059436890 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731059436890Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731059436906 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731059436907 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731059436922 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731059436939 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731059436939Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731059436955 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731059436968 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731059436968Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3663f200: reopening flushed file at 1731059436983 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11049f25: reopening flushed file at 1731059436993 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49092be8: reopening flushed file at 1731059437001 (+8 ms)Finished flush of dataSize ~26.83 KB/27471, heapSize ~33.82 KB/34632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 123ms, sequenceid=72, compaction requested=false at 1731059437011 (+10 ms)Writing region close event to WAL at 1731059437012 (+1 ms)Closed at 1731059437012
2024-11-08T09:50:37,014 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:37,014 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:37,014 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:37,014 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:37,014 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-08T09:50:37,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37415 is added to blk_1073741830_1006 (size=32674)
2024-11-08T09:50:37,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34075 is added to blk_1073741830_1006 (size=32674)
2024-11-08T09:50:37,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43647 is added to blk_1073741830_1006 (size=32674)
2024-11-08T09:50:37,018 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-08T09:50:37,018 INFO  [M:0;cf4d76213909:45843 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-08T09:50:37,018 INFO  [M:0;cf4d76213909:45843 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45843
2024-11-08T09:50:37,019 INFO  [M:0;cf4d76213909:45843 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-08T09:50:37,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:37,130 INFO  [M:0;cf4d76213909:45843 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-08T09:50:37,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45843-0x10119dab9480000, quorum=127.0.0.1:59691, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-08T09:50:37,137 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d404ea1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:37,138 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6732f353{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:37,138 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:37,138 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@243a1ac8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:37,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c6a742f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:37,140 WARN  [BP-1316722941-172.17.0.2-1731059430203 heartbeating to localhost/127.0.0.1:41199 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T09:50:37,140 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T09:50:37,141 WARN  [BP-1316722941-172.17.0.2-1731059430203 heartbeating to localhost/127.0.0.1:41199 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1316722941-172.17.0.2-1731059430203 (Datanode Uuid 4de35748-0b39-4dfe-817e-08e27beb61d8) service to localhost/127.0.0.1:41199
2024-11-08T09:50:37,141 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T09:50:37,141 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data5/current/BP-1316722941-172.17.0.2-1731059430203 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:37,142 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data6/current/BP-1316722941-172.17.0.2-1731059430203 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:37,142 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T09:50:37,145 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@151f492e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:37,146 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@ea7545b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:37,146 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:37,146 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@128490e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:37,146 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f624b91{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:37,148 WARN  [BP-1316722941-172.17.0.2-1731059430203 heartbeating to localhost/127.0.0.1:41199 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T09:50:37,148 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T09:50:37,148 WARN  [BP-1316722941-172.17.0.2-1731059430203 heartbeating to localhost/127.0.0.1:41199 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1316722941-172.17.0.2-1731059430203 (Datanode Uuid a2953b2b-8d80-48a1-bf02-cef6acedcb8b) service to localhost/127.0.0.1:41199
2024-11-08T09:50:37,148 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T09:50:37,149 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data3/current/BP-1316722941-172.17.0.2-1731059430203 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:37,149 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data4/current/BP-1316722941-172.17.0.2-1731059430203 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:37,149 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T09:50:37,151 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3677c717{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-08T09:50:37,151 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11762ce6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:37,151 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:37,151 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@447c571b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:37,151 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1592a7cf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:37,152 WARN  [BP-1316722941-172.17.0.2-1731059430203 heartbeating to localhost/127.0.0.1:41199 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-08T09:50:37,153 WARN  [BP-1316722941-172.17.0.2-1731059430203 heartbeating to localhost/127.0.0.1:41199 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1316722941-172.17.0.2-1731059430203 (Datanode Uuid 1a8191e9-7592-4fa1-9252-160b3f1fe7e6) service to localhost/127.0.0.1:41199
2024-11-08T09:50:37,153 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-08T09:50:37,153 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-08T09:50:37,153 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data1/current/BP-1316722941-172.17.0.2-1731059430203 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:37,153 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/cluster_f6f1bf14-363f-6dca-fc79-3ccb5beba043/data/data2/current/BP-1316722941-172.17.0.2-1731059430203 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-08T09:50:37,154 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-08T09:50:37,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b02112e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-08T09:50:37,159 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@365b31e8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-08T09:50:37,159 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-08T09:50:37,160 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dabbb4f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-08T09:50:37,160 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1120ea6a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/ff8ca830-ca5c-4ecb-fff4-23ef3ab59f13/hadoop.log.dir/,STOPPED}
2024-11-08T09:50:37,166 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-08T09:50:37,191 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-08T09:50:37,198 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestHBaseWalOnEC#testReadWrite[1] Thread=169 (was 105) - Thread LEAK? -, OpenFileDescriptor=526 (was 374) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=240 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=4348 (was 4719)