2024-12-09 17:04:06,162 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc
2024-12-09 17:04:06,178 main DEBUG Took 0.012929 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-09 17:04:06,178 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-09 17:04:06,179 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-09 17:04:06,180 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-09 17:04:06,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,189 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-09 17:04:06,202 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,205 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,206 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,207 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,209 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,209 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,210 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,211 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,211 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,212 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,212 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,213 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,214 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,215 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,216 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-09 17:04:06,217 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,217 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-09 17:04:06,219 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-09 17:04:06,220 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-09 17:04:06,223 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-09 17:04:06,223 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-09 17:04:06,225 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-09 17:04:06,225 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-09 17:04:06,235 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-09 17:04:06,238 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-09 17:04:06,240 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-09 17:04:06,240 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-09 17:04:06,241 main DEBUG createAppenders(={Console})
2024-12-09 17:04:06,242 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc initialized
2024-12-09 17:04:06,242 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc
2024-12-09 17:04:06,242 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc OK.
2024-12-09 17:04:06,243 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-09 17:04:06,244 main DEBUG OutputStream closed
2024-12-09 17:04:06,244 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-09 17:04:06,244 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-09 17:04:06,245 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53ce1329 OK
2024-12-09 17:04:06,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-09 17:04:06,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-09 17:04:06,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-09 17:04:06,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-09 17:04:06,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-09 17:04:06,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-09 17:04:06,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-09 17:04:06,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-09 17:04:06,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-09 17:04:06,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-09 17:04:06,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-09 17:04:06,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-09 17:04:06,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-09 17:04:06,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-09 17:04:06,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-09 17:04:06,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-09 17:04:06,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-09 17:04:06,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-09 17:04:06,344 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-09 17:04:06,345 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-09 17:04:06,345 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-09 17:04:06,346 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-09T17:04:06,562 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d
2024-12-09 17:04:06,565 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-09 17:04:06,565 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-09T17:04:06,573 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-09T17:04:06,613 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=286, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=5882
2024-12-09T17:04:06,616 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-09T17:04:06,620 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6, deleteOnExit=true
2024-12-09T17:04:06,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-09T17:04:06,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/test.cache.data in system properties and HBase conf
2024-12-09T17:04:06,622 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.tmp.dir in system properties and HBase conf
2024-12-09T17:04:06,623 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir in system properties and HBase conf
2024-12-09T17:04:06,623 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-09T17:04:06,624 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-09T17:04:06,624 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-09T17:04:06,726 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-09T17:04:06,807 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-09T17:04:06,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:04:06,811 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:04:06,811 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-09T17:04:06,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:04:06,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-09T17:04:06,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-09T17:04:06,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:04:06,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:04:06,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-09T17:04:06,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/nfs.dump.dir in system properties and HBase conf
2024-12-09T17:04:06,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/java.io.tmpdir in system properties and HBase conf
2024-12-09T17:04:06,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:04:06,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-09T17:04:06,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-09T17:04:07,223 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:04:07,834 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-09T17:04:07,894 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2313ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-09T17:04:07,953 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:04:08,013 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:04:08,030 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:04:08,030 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:04:08,032 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:04:08,043 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:04:08,045 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:04:08,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:04:08,235 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5682c4d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/java.io.tmpdir/jetty-localhost-34125-hadoop-hdfs-3_4_1-tests_jar-_-any-771929378455534973/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:04:08,241 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:34125}
2024-12-09T17:04:08,241 INFO  [Time-limited test {}] server.Server(415): Started @2661ms
2024-12-09T17:04:08,269 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:04:08,714 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:04:08,721 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:04:08,723 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:04:08,723 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:04:08,723 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:04:08,724 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:04:08,724 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:04:08,830 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6aad8790{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/java.io.tmpdir/jetty-localhost-39829-hadoop-hdfs-3_4_1-tests_jar-_-any-12165837888968798621/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:04:08,831 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:39829}
2024-12-09T17:04:08,831 INFO  [Time-limited test {}] server.Server(415): Started @3251ms
2024-12-09T17:04:08,879 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:04:08,998 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:04:09,004 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:04:09,005 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:04:09,005 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:04:09,005 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:04:09,007 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:04:09,008 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:04:09,113 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@163cfad6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/java.io.tmpdir/jetty-localhost-37645-hadoop-hdfs-3_4_1-tests_jar-_-any-9372705651295430034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:04:09,113 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:37645}
2024-12-09T17:04:09,114 INFO  [Time-limited test {}] server.Server(415): Started @3533ms
2024-12-09T17:04:09,115 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:04:10,005 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data3/current/BP-2139196168-172.17.0.2-1733763847296/current, will proceed with Du for space computation calculation, 
2024-12-09T17:04:10,005 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data4/current/BP-2139196168-172.17.0.2-1733763847296/current, will proceed with Du for space computation calculation, 
2024-12-09T17:04:10,005 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data2/current/BP-2139196168-172.17.0.2-1733763847296/current, will proceed with Du for space computation calculation, 
2024-12-09T17:04:10,005 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data1/current/BP-2139196168-172.17.0.2-1733763847296/current, will proceed with Du for space computation calculation, 
2024-12-09T17:04:10,042 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:04:10,042 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:04:10,086 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2dc3e35a55314a7a with lease ID 0x60830cf365a112f6: Processing first storage report for DS-5080d36b-beea-4542-bfd7-0c239f24ad9e from datanode DatanodeRegistration(127.0.0.1:37875, datanodeUuid=d528a27f-b13f-4ff6-8c0c-00bd168fc3b1, infoPort=39007, infoSecurePort=0, ipcPort=38133, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296)
2024-12-09T17:04:10,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2dc3e35a55314a7a with lease ID 0x60830cf365a112f6: from storage DS-5080d36b-beea-4542-bfd7-0c239f24ad9e node DatanodeRegistration(127.0.0.1:37875, datanodeUuid=d528a27f-b13f-4ff6-8c0c-00bd168fc3b1, infoPort=39007, infoSecurePort=0, ipcPort=38133, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-09T17:04:10,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x710f3dc64eb0e9e with lease ID 0x60830cf365a112f5: Processing first storage report for DS-67feab2d-be29-4569-a0a5-c29148a5982e from datanode DatanodeRegistration(127.0.0.1:44791, datanodeUuid=16401565-5422-4d09-b9b9-c82058ffbcc2, infoPort=33915, infoSecurePort=0, ipcPort=44507, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296)
2024-12-09T17:04:10,087 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x710f3dc64eb0e9e with lease ID 0x60830cf365a112f5: from storage DS-67feab2d-be29-4569-a0a5-c29148a5982e node DatanodeRegistration(127.0.0.1:44791, datanodeUuid=16401565-5422-4d09-b9b9-c82058ffbcc2, infoPort=33915, infoSecurePort=0, ipcPort=44507, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:04:10,088 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2dc3e35a55314a7a with lease ID 0x60830cf365a112f6: Processing first storage report for DS-569a71d3-e1c6-4bd2-96f6-f92961890fcb from datanode DatanodeRegistration(127.0.0.1:37875, datanodeUuid=d528a27f-b13f-4ff6-8c0c-00bd168fc3b1, infoPort=39007, infoSecurePort=0, ipcPort=38133, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296)
2024-12-09T17:04:10,088 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2dc3e35a55314a7a with lease ID 0x60830cf365a112f6: from storage DS-569a71d3-e1c6-4bd2-96f6-f92961890fcb node DatanodeRegistration(127.0.0.1:37875, datanodeUuid=d528a27f-b13f-4ff6-8c0c-00bd168fc3b1, infoPort=39007, infoSecurePort=0, ipcPort=38133, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:04:10,088 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x710f3dc64eb0e9e with lease ID 0x60830cf365a112f5: Processing first storage report for DS-fff1129b-0188-4c80-9489-47bac2da5ce5 from datanode DatanodeRegistration(127.0.0.1:44791, datanodeUuid=16401565-5422-4d09-b9b9-c82058ffbcc2, infoPort=33915, infoSecurePort=0, ipcPort=44507, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296)
2024-12-09T17:04:10,088 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x710f3dc64eb0e9e with lease ID 0x60830cf365a112f5: from storage DS-fff1129b-0188-4c80-9489-47bac2da5ce5 node DatanodeRegistration(127.0.0.1:44791, datanodeUuid=16401565-5422-4d09-b9b9-c82058ffbcc2, infoPort=33915, infoSecurePort=0, ipcPort=44507, storageInfo=lv=-57;cid=testClusterID;nsid=1881112184;c=1733763847296), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:04:10,129 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d
2024-12-09T17:04:10,192 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/zookeeper_0, clientPort=56951, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-09T17:04:10,200 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=56951
2024-12-09T17:04:10,212 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:10,214 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:10,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:04:10,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:04:10,839 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476 with version=8
2024-12-09T17:04:10,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase-staging
2024-12-09T17:04:10,937 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-09T17:04:11,170 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:04:11,185 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:04:11,185 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:04:11,185 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:04:11,186 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:04:11,186 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:04:11,303 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:04:11,352 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-09T17:04:11,360 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-09T17:04:11,363 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:04:11,384 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5200 (auto-detected)
2024-12-09T17:04:11,385 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-09T17:04:11,400 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36053
2024-12-09T17:04:11,406 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:11,408 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:11,419 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36053 connecting to ZooKeeper ensemble=127.0.0.1:56951
2024-12-09T17:04:11,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360530x0, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:04:11,530 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36053-0x1000bbffc400000 connected
2024-12-09T17:04:11,615 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:04:11,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:04:11,624 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:04:11,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36053
2024-12-09T17:04:11,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36053
2024-12-09T17:04:11,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36053
2024-12-09T17:04:11,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36053
2024-12-09T17:04:11,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36053
2024-12-09T17:04:11,636 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476, hbase.cluster.distributed=false
2024-12-09T17:04:11,700 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:04:11,700 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:04:11,701 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:04:11,701 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:04:11,701 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:04:11,701 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:04:11,703 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:04:11,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:04:11,706 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41951
2024-12-09T17:04:11,707 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:04:11,712 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:04:11,713 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:11,716 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:11,720 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:41951 connecting to ZooKeeper ensemble=127.0.0.1:56951
2024-12-09T17:04:11,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419510x0, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:04:11,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419510x0, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:04:11,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41951-0x1000bbffc400001 connected
2024-12-09T17:04:11,731 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:04:11,732 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:04:11,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41951
2024-12-09T17:04:11,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41951
2024-12-09T17:04:11,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41951
2024-12-09T17:04:11,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41951
2024-12-09T17:04:11,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41951
2024-12-09T17:04:11,738 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/80c69eb3c456,36053,1733763850932
2024-12-09T17:04:11,750 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80c69eb3c456:36053
2024-12-09T17:04:11,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:04:11,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:04:11,753 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80c69eb3c456,36053,1733763850932
2024-12-09T17:04:11,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:04:11,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:04:11,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:11,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:11,780 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:04:11,781 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80c69eb3c456,36053,1733763850932 from backup master directory
2024-12-09T17:04:11,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:04:11,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80c69eb3c456,36053,1733763850932
2024-12-09T17:04:11,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:04:11,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:04:11,788 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:04:11,788 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80c69eb3c456,36053,1733763850932
2024-12-09T17:04:11,790 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-09T17:04:11,791 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-09T17:04:11,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:04:11,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:04:11,862 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase.id with ID: f085c9e5-f80c-40ef-b99d-74788d693078
2024-12-09T17:04:11,903 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:11,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:11,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:11,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:04:11,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:04:11,960 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:04:11,962 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-09T17:04:11,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:04:11,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:04:11,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:04:12,011 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store
2024-12-09T17:04:12,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:04:12,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:04:12,031 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-09T17:04:12,031 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:12,032 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:04:12,033 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:04:12,033 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:04:12,033 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:04:12,033 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:04:12,033 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:04:12,033 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:04:12,035 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/.initializing
2024-12-09T17:04:12,036 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/WALs/80c69eb3c456,36053,1733763850932
2024-12-09T17:04:12,049 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C36053%2C1733763850932, suffix=, logDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/WALs/80c69eb3c456,36053,1733763850932, archiveDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/oldWALs, maxLogs=10
2024-12-09T17:04:12,060 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36053%2C1733763850932.1733763852058
2024-12-09T17:04:12,061 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag.
2024-12-09T17:04:12,062 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag.
2024-12-09T17:04:12,079 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/WALs/80c69eb3c456,36053,1733763850932/80c69eb3c456%2C36053%2C1733763850932.1733763852058
2024-12-09T17:04:12,087 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:04:12,087 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:04:12,088 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:12,091 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:04:12,093 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:04:12,126 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:04:12,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-09T17:04:12,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,154 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:12,154 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:04:12,158 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-09T17:04:12,158 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,159 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:04:12,160 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:04:12,163 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-09T17:04:12,163 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,164 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:04:12,165 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:04:12,167 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-09T17:04:12,167 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,169 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:04:12,172 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:04:12,174 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:04:12,183 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-09T17:04:12,186 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:04:12,191 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:04:12,192 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=871654, jitterRate=0.10836613178253174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-09T17:04:12,195 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:04:12,197 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-09T17:04:12,222 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351b704e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:04:12,252 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-09T17:04:12,263 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-09T17:04:12,263 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-09T17:04:12,265 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-09T17:04:12,266 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec
2024-12-09T17:04:12,271 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec
2024-12-09T17:04:12,271 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-09T17:04:12,295 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-09T17:04:12,306 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-09T17:04:12,312 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-09T17:04:12,314 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-09T17:04:12,316 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-09T17:04:12,326 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-09T17:04:12,328 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-09T17:04:12,331 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-09T17:04:12,337 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-09T17:04:12,338 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-09T17:04:12,345 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-09T17:04:12,354 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-09T17:04:12,362 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-09T17:04:12,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:04:12,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:04:12,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:12,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:12,371 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=80c69eb3c456,36053,1733763850932, sessionid=0x1000bbffc400000, setting cluster-up flag (Was=false)
2024-12-09T17:04:12,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:12,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:12,420 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-09T17:04:12,423 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,36053,1733763850932
2024-12-09T17:04:12,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:12,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:12,470 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-09T17:04:12,473 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,36053,1733763850932
2024-12-09T17:04:12,540 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-09T17:04:12,546 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-09T17:04:12,549 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-09T17:04:12,553 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80c69eb3c456:41951
2024-12-09T17:04:12,554 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1008): ClusterId : f085c9e5-f80c-40ef-b99d-74788d693078
2024-12-09T17:04:12,554 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80c69eb3c456,36053,1733763850932
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-09T17:04:12,557 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:04:12,557 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:04:12,557 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:04:12,557 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:04:12,557 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:04:12,557 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80c69eb3c456:0, corePoolSize=10, maxPoolSize=10
2024-12-09T17:04:12,557 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,558 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:04:12,558 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,559 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733763882559
2024-12-09T17:04:12,560 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-09T17:04:12,561 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-09T17:04:12,562 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:04:12,563 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-09T17:04:12,565 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-09T17:04:12,565 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-09T17:04:12,565 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-09T17:04:12,565 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-09T17:04:12,566 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,566 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,567 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:04:12,567 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-09T17:04:12,568 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-09T17:04:12,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-09T17:04:12,570 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:04:12,570 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:04:12,571 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-09T17:04:12,571 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-09T17:04:12,572 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733763852572,5,FailOnTimeoutGroup]
2024-12-09T17:04:12,573 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733763852572,5,FailOnTimeoutGroup]
2024-12-09T17:04:12,573 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,573 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-09T17:04:12,574 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,574 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:04:12,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:04:12,578 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-09T17:04:12,578 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476
2024-12-09T17:04:12,580 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:04:12,580 DEBUG [RS:0;80c69eb3c456:41951 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb6019b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:04:12,582 DEBUG [RS:0;80c69eb3c456:41951 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3fef83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:04:12,585 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:04:12,585 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:04:12,586 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:04:12,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:04:12,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:04:12,588 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,36053,1733763850932 with isa=80c69eb3c456/172.17.0.2:41951, startcode=1733763851699
2024-12-09T17:04:12,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:12,591 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:04:12,594 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:04:12,594 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,595 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:12,595 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:04:12,597 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:04:12,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,598 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:12,598 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:04:12,600 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:04:12,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:12,601 DEBUG [RS:0;80c69eb3c456:41951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:04:12,602 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:12,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740
2024-12-09T17:04:12,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740
2024-12-09T17:04:12,606 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:04:12,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:04:12,612 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:04:12,613 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834696, jitterRate=0.06137128174304962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:04:12,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:04:12,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:04:12,615 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:04:12,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:04:12,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:04:12,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:04:12,617 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:04:12,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:04:12,620 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:04:12,620 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-09T17:04:12,626 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-09T17:04:12,633 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-09T17:04:12,635 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-09T17:04:12,669 INFO  [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49245, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:04:12,674 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36053 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,41951,1733763851699
2024-12-09T17:04:12,676 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36053 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:12,689 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476
2024-12-09T17:04:12,689 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:44739
2024-12-09T17:04:12,689 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:04:12,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:04:12,704 DEBUG [RS:0;80c69eb3c456:41951 {}] zookeeper.ZKUtil(111): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,41951,1733763851699
2024-12-09T17:04:12,704 WARN  [RS:0;80c69eb3c456:41951 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:04:12,705 INFO  [RS:0;80c69eb3c456:41951 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:04:12,705 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699
2024-12-09T17:04:12,707 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,41951,1733763851699]
2024-12-09T17:04:12,722 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:04:12,730 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:04:12,741 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:04:12,743 INFO  [RS:0;80c69eb3c456:41951 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:04:12,743 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,744 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:04:12,749 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,750 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,751 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:04:12,751 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:04:12,751 DEBUG [RS:0;80c69eb3c456:41951 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:04:12,752 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,752 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,752 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,752 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,752 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41951,1733763851699-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:04:12,767 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:04:12,768 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41951,1733763851699-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:12,782 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.Replication(204): 80c69eb3c456,41951,1733763851699 started
2024-12-09T17:04:12,783 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,41951,1733763851699, RpcServer on 80c69eb3c456/172.17.0.2:41951, sessionid=0x1000bbffc400001
2024-12-09T17:04:12,783 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:04:12,783 DEBUG [RS:0;80c69eb3c456:41951 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,41951,1733763851699
2024-12-09T17:04:12,783 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,41951,1733763851699'
2024-12-09T17:04:12,783 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:04:12,784 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:04:12,785 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:04:12,785 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:04:12,785 DEBUG [RS:0;80c69eb3c456:41951 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,41951,1733763851699
2024-12-09T17:04:12,785 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,41951,1733763851699'
2024-12-09T17:04:12,785 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:04:12,786 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:04:12,786 WARN  [80c69eb3c456:36053 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-09T17:04:12,786 DEBUG [RS:0;80c69eb3c456:41951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:04:12,786 INFO  [RS:0;80c69eb3c456:41951 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:04:12,786 INFO  [RS:0;80c69eb3c456:41951 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:04:12,898 INFO  [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C41951%2C1733763851699, suffix=, logDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699, archiveDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs, maxLogs=32
2024-12-09T17:04:12,901 INFO  [RS:0;80c69eb3c456:41951 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763852901
2024-12-09T17:04:12,909 INFO  [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763852901
2024-12-09T17:04:12,909 DEBUG [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:04:13,037 DEBUG [80c69eb3c456:36053 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-09T17:04:13,041 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:13,045 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,41951,1733763851699, state=OPENING
2024-12-09T17:04:13,087 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-09T17:04:13,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:13,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:13,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:04:13,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:04:13,100 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=80c69eb3c456,41951,1733763851699}]
2024-12-09T17:04:13,281 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,41951,1733763851699
2024-12-09T17:04:13,282 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-09T17:04:13,285 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48416, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-09T17:04:13,296 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-09T17:04:13,297 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:04:13,300 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C41951%2C1733763851699.meta, suffix=.meta, logDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699, archiveDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs, maxLogs=32
2024-12-09T17:04:13,302 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.meta.1733763853302.meta
2024-12-09T17:04:13,310 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.meta.1733763853302.meta
2024-12-09T17:04:13,311 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:33915:33915)]
2024-12-09T17:04:13,311 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:04:13,312 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-09T17:04:13,364 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-09T17:04:13,368 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-09T17:04:13,371 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-09T17:04:13,371 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:13,372 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-09T17:04:13,372 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-09T17:04:13,374 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:04:13,376 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:04:13,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:13,377 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:13,377 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:04:13,378 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:04:13,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:13,379 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:13,379 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:04:13,381 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:04:13,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:13,381 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:04:13,383 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740
2024-12-09T17:04:13,385 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740
2024-12-09T17:04:13,388 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:04:13,390 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:04:13,392 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756570, jitterRate=-0.037971630692481995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:04:13,393 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:04:13,399 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733763853275
2024-12-09T17:04:13,408 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-09T17:04:13,409 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-09T17:04:13,409 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:13,411 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,41951,1733763851699, state=OPEN
2024-12-09T17:04:13,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:04:13,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:04:13,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:04:13,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:04:13,479 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-09T17:04:13,480 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=80c69eb3c456,41951,1733763851699 in 372 msec
2024-12-09T17:04:13,486 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-09T17:04:13,486 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 855 msec
2024-12-09T17:04:13,492 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 982 msec
2024-12-09T17:04:13,493 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733763853492, completionTime=-1
2024-12-09T17:04:13,493 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-09T17:04:13,493 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-09T17:04:13,524 DEBUG [hconnection-0x558380c6-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:04:13,526 INFO  [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48426, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:04:13,534 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-09T17:04:13,534 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733763913534
2024-12-09T17:04:13,535 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733763973535
2024-12-09T17:04:13,535 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 41 msec
2024-12-09T17:04:13,579 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36053,1733763850932-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:13,579 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36053,1733763850932-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:13,579 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36053,1733763850932-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:13,580 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80c69eb3c456:36053, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:13,580 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:13,586 DEBUG [master/80c69eb3c456:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-09T17:04:13,589 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-09T17:04:13,590 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:04:13,595 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-09T17:04:13,598 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:04:13,599 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:13,601 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:04:13,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:04:13,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:04:13,614 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d9eb5e5e026495891b2ae9d1a368dda7, NAME => 'hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476
2024-12-09T17:04:13,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:04:13,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:04:13,625 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:13,625 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing d9eb5e5e026495891b2ae9d1a368dda7, disabling compactions & flushes
2024-12-09T17:04:13,625 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:13,625 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:13,625 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7. after waiting 0 ms
2024-12-09T17:04:13,625 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:13,625 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:13,625 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for d9eb5e5e026495891b2ae9d1a368dda7:

2024-12-09T17:04:13,627 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:04:13,632 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733763853628"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733763853628"}]},"ts":"1733763853628"}
2024-12-09T17:04:13,651 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:04:13,653 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:04:13,656 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763853654"}]},"ts":"1733763853654"}
2024-12-09T17:04:13,661 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-09T17:04:13,680 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=d9eb5e5e026495891b2ae9d1a368dda7, ASSIGN}]
2024-12-09T17:04:13,683 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=d9eb5e5e026495891b2ae9d1a368dda7, ASSIGN
2024-12-09T17:04:13,685 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=d9eb5e5e026495891b2ae9d1a368dda7, ASSIGN; state=OFFLINE, location=80c69eb3c456,41951,1733763851699; forceNewPlan=false, retain=false
2024-12-09T17:04:13,836 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=d9eb5e5e026495891b2ae9d1a368dda7, regionState=OPENING, regionLocation=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:13,844 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure d9eb5e5e026495891b2ae9d1a368dda7, server=80c69eb3c456,41951,1733763851699}]
2024-12-09T17:04:13,998 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,41951,1733763851699
2024-12-09T17:04:14,006 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:14,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => d9eb5e5e026495891b2ae9d1a368dda7, NAME => 'hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:04:14,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:04:14,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:14,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:04:14,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:04:14,011 INFO  [StoreOpener-d9eb5e5e026495891b2ae9d1a368dda7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d9eb5e5e026495891b2ae9d1a368dda7 
2024-12-09T17:04:14,013 INFO  [StoreOpener-d9eb5e5e026495891b2ae9d1a368dda7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9eb5e5e026495891b2ae9d1a368dda7 columnFamilyName info
2024-12-09T17:04:14,014 DEBUG [StoreOpener-d9eb5e5e026495891b2ae9d1a368dda7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:14,014 INFO  [StoreOpener-d9eb5e5e026495891b2ae9d1a368dda7-1 {}] regionserver.HStore(327): Store=d9eb5e5e026495891b2ae9d1a368dda7/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:04:14,016 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:04:14,017 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:04:14,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:04:14,024 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:04:14,026 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened d9eb5e5e026495891b2ae9d1a368dda7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=731500, jitterRate=-0.06985042989253998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:04:14,027 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for d9eb5e5e026495891b2ae9d1a368dda7:

2024-12-09T17:04:14,029 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7., pid=6, masterSystemTime=1733763853998
2024-12-09T17:04:14,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:14,032 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:04:14,033 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=d9eb5e5e026495891b2ae9d1a368dda7, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:14,039 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-09T17:04:14,040 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure d9eb5e5e026495891b2ae9d1a368dda7, server=80c69eb3c456,41951,1733763851699 in 193 msec
2024-12-09T17:04:14,043 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-09T17:04:14,043 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=d9eb5e5e026495891b2ae9d1a368dda7, ASSIGN in 359 msec
2024-12-09T17:04:14,044 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:04:14,044 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763854044"}]},"ts":"1733763854044"}
2024-12-09T17:04:14,047 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-09T17:04:14,090 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:04:14,095 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 500 msec
2024-12-09T17:04:14,099 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-09T17:04:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:04:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:04:14,145 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-09T17:04:14,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:04:14,186 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 41 msec
2024-12-09T17:04:14,201 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-09T17:04:14,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:04:14,243 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 40 msec
2024-12-09T17:04:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-09T17:04:14,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-09T17:04:14,296 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 2.507sec
2024-12-09T17:04:14,297 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-09T17:04:14,299 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-09T17:04:14,301 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-09T17:04:14,302 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-09T17:04:14,303 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-09T17:04:14,304 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36053,1733763850932-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:04:14,304 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36053,1733763850932-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-09T17:04:14,310 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-09T17:04:14,311 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-09T17:04:14,311 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36053,1733763850932-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:04:14,356 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x478880db to 127.0.0.1:56951 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16120bed
2024-12-09T17:04:14,356 WARN  [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry
2024-12-09T17:04:14,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afc2bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:04:14,380 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-09T17:04:14,380 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-09T17:04:14,389 DEBUG [hconnection-0x4642e44e-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:04:14,441 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48432, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:04:14,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=80c69eb3c456,36053,1733763850932
2024-12-09T17:04:14,452 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:04:14,458 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-09T17:04:14,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-09T17:04:14,467 INFO  [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41930, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-09T17:04:14,472 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-09T17:04:14,472 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-09T17:04:14,475 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:04:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-09T17:04:14,479 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:04:14,479 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:14,481 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:04:14,481 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 9
2024-12-09T17:04:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:04:14,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741837_1013 (size=389)
2024-12-09T17:04:14,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741837_1013 (size=389)
2024-12-09T17:04:14,495 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => fa1751ee8a2b3d89324db0b2e928149c, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476
2024-12-09T17:04:14,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741838_1014 (size=72)
2024-12-09T17:04:14,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741838_1014 (size=72)
2024-12-09T17:04:14,506 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:14,506 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing fa1751ee8a2b3d89324db0b2e928149c, disabling compactions & flushes
2024-12-09T17:04:14,506 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,506 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,506 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c. after waiting 0 ms
2024-12-09T17:04:14,507 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,507 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,507 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:04:14,508 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:04:14,509 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733763854508"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733763854508"}]},"ts":"1733763854508"}
2024-12-09T17:04:14,511 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:04:14,513 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:04:14,513 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763854513"}]},"ts":"1733763854513"}
2024-12-09T17:04:14,516 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-09T17:04:14,537 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fa1751ee8a2b3d89324db0b2e928149c, ASSIGN}]
2024-12-09T17:04:14,539 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fa1751ee8a2b3d89324db0b2e928149c, ASSIGN
2024-12-09T17:04:14,540 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fa1751ee8a2b3d89324db0b2e928149c, ASSIGN; state=OFFLINE, location=80c69eb3c456,41951,1733763851699; forceNewPlan=false, retain=false
2024-12-09T17:04:14,691 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fa1751ee8a2b3d89324db0b2e928149c, regionState=OPENING, regionLocation=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:14,698 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure fa1751ee8a2b3d89324db0b2e928149c, server=80c69eb3c456,41951,1733763851699}]
2024-12-09T17:04:14,854 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,41951,1733763851699
2024-12-09T17:04:14,862 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,862 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => fa1751ee8a2b3d89324db0b2e928149c, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:04:14,863 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:14,863 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:04:14,863 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:14,863 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:14,866 INFO  [StoreOpener-fa1751ee8a2b3d89324db0b2e928149c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fa1751ee8a2b3d89324db0b2e928149c 
2024-12-09T17:04:14,869 INFO  [StoreOpener-fa1751ee8a2b3d89324db0b2e928149c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa1751ee8a2b3d89324db0b2e928149c columnFamilyName info
2024-12-09T17:04:14,869 DEBUG [StoreOpener-fa1751ee8a2b3d89324db0b2e928149c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:04:14,870 INFO  [StoreOpener-fa1751ee8a2b3d89324db0b2e928149c-1 {}] regionserver.HStore(327): Store=fa1751ee8a2b3d89324db0b2e928149c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:04:14,871 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:14,872 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:14,875 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:14,878 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:04:14,879 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened fa1751ee8a2b3d89324db0b2e928149c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=774874, jitterRate=-0.01469700038433075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:04:14,880 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:04:14,881 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c., pid=11, masterSystemTime=1733763854853
2024-12-09T17:04:14,884 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,884 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:14,885 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fa1751ee8a2b3d89324db0b2e928149c, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,41951,1733763851699
2024-12-09T17:04:14,891 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-09T17:04:14,892 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure fa1751ee8a2b3d89324db0b2e928149c, server=80c69eb3c456,41951,1733763851699 in 190 msec
2024-12-09T17:04:14,894 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-09T17:04:14,894 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fa1751ee8a2b3d89324db0b2e928149c, ASSIGN in 354 msec
2024-12-09T17:04:14,895 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:04:14,896 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763854895"}]},"ts":"1733763854895"}
2024-12-09T17:04:14,898 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-09T17:04:14,939 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:04:14,943 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 464 msec
2024-12-09T17:04:18,881 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-09T17:04:18,923 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-09T17:04:18,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-09T17:04:18,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-09T17:04:21,350 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-09T17:04:21,351 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-09T17:04:21,355 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-09T17:04:21,355 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-09T17:04:21,358 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace
2024-12-09T17:04:21,358 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer
2024-12-09T17:04:21,359 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-09T17:04:21,360 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-09T17:04:21,360 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-09T17:04:21,360 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-09T17:04:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:04:24,496 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling, procId: 9 completed
2024-12-09T17:04:24,500 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-09T17:04:24,501 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:04:24,502 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763864502
2024-12-09T17:04:24,513 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763852901 with entries=4, filesize=947 B; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763864502
2024-12-09T17:04:24,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:04:24,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763852901 is not closed yet, will try archiving it next time
2024-12-09T17:04:24,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741833_1009 (size=955)
2024-12-09T17:04:24,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741833_1009 (size=955)
2024-12-09T17:04:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41951 {}] regionserver.HRegion(8581): Flush requested on fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:36,567 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fa1751ee8a2b3d89324db0b2e928149c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:04:36,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/a2188f2f78c44e7184ea7e0bed76b032 is 1080, key is row0001/info:/1733763864519/Put/seqid=0
2024-12-09T17:04:36,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741840_1016 (size=12509)
2024-12-09T17:04:36,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741840_1016 (size=12509)
2024-12-09T17:04:36,628 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/a2188f2f78c44e7184ea7e0bed76b032
2024-12-09T17:04:36,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/a2188f2f78c44e7184ea7e0bed76b032 as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032
2024-12-09T17:04:36,680 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032, entries=7, sequenceid=11, filesize=12.2 K
2024-12-09T17:04:36,683 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fa1751ee8a2b3d89324db0b2e928149c in 116ms, sequenceid=11, compaction requested=false
2024-12-09T17:04:36,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:04:40,127 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:04:44,582 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763884581
2024-12-09T17:04:44,798 INFO  [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 211 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:44,800 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763864502 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763884581
2024-12-09T17:04:44,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:04:44,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763864502 is not closed yet, will try archiving it next time
2024-12-09T17:04:44,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741839_1015 (size=12399)
2024-12-09T17:04:44,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741839_1015 (size=12399)
2024-12-09T17:04:45,006 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:46,586 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:04:46,589 INFO  [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48682, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:04:47,211 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:49,416 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:51,621 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41951 {}] regionserver.HRegion(8581): Flush requested on fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:04:51,622 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fa1751ee8a2b3d89324db0b2e928149c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:04:51,826 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:51,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/8eca4e2feac34335b96f10c26549f5da is 1080, key is row0008/info:/1733763878569/Put/seqid=0
2024-12-09T17:04:51,845 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741842_1018 (size=12509)
2024-12-09T17:04:51,845 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741842_1018 (size=12509)
2024-12-09T17:04:51,846 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/8eca4e2feac34335b96f10c26549f5da
2024-12-09T17:04:51,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/8eca4e2feac34335b96f10c26549f5da as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/8eca4e2feac34335b96f10c26549f5da
2024-12-09T17:04:51,867 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/8eca4e2feac34335b96f10c26549f5da, entries=7, sequenceid=21, filesize=12.2 K
2024-12-09T17:04:52,070 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:52,070 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fa1751ee8a2b3d89324db0b2e928149c in 448ms, sequenceid=21, compaction requested=false
2024-12-09T17:04:52,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:04:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=24.4 K, sizeToCheck=16.0 K
2024-12-09T17:04:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:04:52,074 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032 because midkey is the same as first or last row
2024-12-09T17:04:53,828 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:54,323 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-09T17:04:54,323 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-09T17:04:56,035 WARN  [sync.1 {}] wal.AbstractFSWAL(1346): Requesting log roll because we exceeded slow sync threshold; count=7, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:56,039 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C41951%2C1733763851699:(num 1733763884581) roll requested
2024-12-09T17:04:56,039 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:56,040 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763896039
2024-12-09T17:04:56,254 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 211 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:56,455 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:04:56,457 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763884581 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763896039
2024-12-09T17:04:56,457 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:33915:33915)]
2024-12-09T17:04:56,457 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763884581 is not closed yet, will try archiving it next time
2024-12-09T17:04:56,460 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763864502 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763864502
2024-12-09T17:04:56,462 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741841_1017 (size=7739)
2024-12-09T17:04:56,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741841_1017 (size=7739)
2024-12-09T17:04:58,242 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:04:59,863 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region fa1751ee8a2b3d89324db0b2e928149c, had cached 0 bytes from a total of 25018
2024-12-09T17:05:00,447 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:02,652 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:04,858 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:06,862 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-09T17:05:06,863 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763906863
2024-12-09T17:05:10,127 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:05:11,881 INFO  [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 5010 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:11,881 WARN  [Time-limited test {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5010 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:11,881 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C41951%2C1733763851699:(num 1733763906863) roll requested
2024-12-09T17:05:13,588 DEBUG [master/80c69eb3c456:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region d9eb5e5e026495891b2ae9d1a368dda7 changed from -1.0 to 0.0, refreshing cache
2024-12-09T17:05:16,882 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:16,882 WARN  [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:16,884 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763896039 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763906863
2024-12-09T17:05:16,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:05:16,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763896039 is not closed yet, will try archiving it next time
2024-12-09T17:05:16,885 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763916885
2024-12-09T17:05:16,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741843_1019 (size=4753)
2024-12-09T17:05:16,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741843_1019 (size=4753)
2024-12-09T17:05:21,890 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:21,890 WARN  [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41951 {}] regionserver.HRegion(8581): Flush requested on fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:05:21,891 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fa1751ee8a2b3d89324db0b2e928149c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:05:21,899 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:21,899 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:23,892 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-09T17:05:26,893 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:26,894 WARN  [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:26,899 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:26,900 WARN  [sync.0 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:26,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/a29f530dcc1447d88ff8a6bd6e71eefc is 1080, key is row0015/info:/1733763893625/Put/seqid=0
2024-12-09T17:05:26,901 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763906863 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763916885
2024-12-09T17:05:26,901 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:33915:33915)]
2024-12-09T17:05:26,901 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763906863 is not closed yet, will try archiving it next time
2024-12-09T17:05:26,901 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C41951%2C1733763851699:(num 1733763916885) roll requested
2024-12-09T17:05:26,902 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763926902
2024-12-09T17:05:26,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741844_1020 (size=1569)
2024-12-09T17:05:26,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741844_1020 (size=1569)
2024-12-09T17:05:26,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741846_1022 (size=12509)
2024-12-09T17:05:26,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741846_1022 (size=12509)
2024-12-09T17:05:26,910 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/a29f530dcc1447d88ff8a6bd6e71eefc
2024-12-09T17:05:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/a29f530dcc1447d88ff8a6bd6e71eefc as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a29f530dcc1447d88ff8a6bd6e71eefc
2024-12-09T17:05:26,929 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a29f530dcc1447d88ff8a6bd6e71eefc, entries=7, sequenceid=31, filesize=12.2 K
2024-12-09T17:05:31,931 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:31,931 WARN  [sync.1 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:31,931 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fa1751ee8a2b3d89324db0b2e928149c in 10041ms, sequenceid=31, compaction requested=true
2024-12-09T17:05:31,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:05:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=36.6 K, sizeToCheck=16.0 K
2024-12-09T17:05:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:05:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032 because midkey is the same as first or last row
2024-12-09T17:05:31,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa1751ee8a2b3d89324db0b2e928149c:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:05:31,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:05:31,935 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5030 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:31,935 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5030 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:31,935 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:05:31,939 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:05:31,941 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.HStore(1540): fa1751ee8a2b3d89324db0b2e928149c/info is initiating minor compaction (all files)
2024-12-09T17:05:31,941 INFO  [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of fa1751ee8a2b3d89324db0b2e928149c/info in TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:05:31,942 INFO  [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/8eca4e2feac34335b96f10c26549f5da, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a29f530dcc1447d88ff8a6bd6e71eefc] into tmpdir=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp, totalSize=36.6 K
2024-12-09T17:05:31,943 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] compactions.Compactor(224): Compacting a2188f2f78c44e7184ea7e0bed76b032, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733763864519
2024-12-09T17:05:31,944 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8eca4e2feac34335b96f10c26549f5da, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733763878569
2024-12-09T17:05:31,944 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] compactions.Compactor(224): Compacting a29f530dcc1447d88ff8a6bd6e71eefc, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733763893625
2024-12-09T17:05:31,969 INFO  [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa1751ee8a2b3d89324db0b2e928149c#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:05:31,970 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/0c46de420a264f29a72e8596fc7ecc35 is 1080, key is row0001/info:/1733763864519/Put/seqid=0
2024-12-09T17:05:31,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741848_1024 (size=27710)
2024-12-09T17:05:31,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741848_1024 (size=27710)
2024-12-09T17:05:31,986 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/0c46de420a264f29a72e8596fc7ecc35 as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/0c46de420a264f29a72e8596fc7ecc35
2024-12-09T17:05:36,936 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:36,936 WARN  [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK], DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK]]
2024-12-09T17:05:36,937 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763916885 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763926902
2024-12-09T17:05:36,938 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:05:36,938 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763916885 is not closed yet, will try archiving it next time
2024-12-09T17:05:36,939 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763884581 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763884581
2024-12-09T17:05:36,939 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C41951%2C1733763851699:(num 1733763936939) roll requested
2024-12-09T17:05:36,940 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763936939
2024-12-09T17:05:36,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741845_1021 (size=438)
2024-12-09T17:05:36,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741845_1021 (size=438)
2024-12-09T17:05:36,974 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763896039 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763896039
2024-12-09T17:05:36,976 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763906863 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763906863
2024-12-09T17:05:36,978 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763916885 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763916885
2024-12-09T17:05:40,128 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:05:41,973 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5034 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:41,973 WARN  [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5034 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:41,977 INFO  [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in fa1751ee8a2b3d89324db0b2e928149c/info of fa1751ee8a2b3d89324db0b2e928149c into 0c46de420a264f29a72e8596fc7ecc35(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 10sec to execute.
2024-12-09T17:05:41,977 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:05:41,977 INFO  [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c., storeName=fa1751ee8a2b3d89324db0b2e928149c/info, priority=13, startTime=1733763931934; duration=10sec
2024-12-09T17:05:41,979 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=27.1 K, sizeToCheck=16.0 K
2024-12-09T17:05:41,979 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:05:41,979 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/0c46de420a264f29a72e8596fc7ecc35 because midkey is the same as first or last row
2024-12-09T17:05:41,979 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:05:41,979 DEBUG [RS:0;80c69eb3c456:41951-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa1751ee8a2b3d89324db0b2e928149c:info
2024-12-09T17:05:41,981 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:41,981 WARN  [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44791,DS-67feab2d-be29-4569-a0a5-c29148a5982e,DISK], DatanodeInfoWithStorage[127.0.0.1:37875,DS-5080d36b-beea-4542-bfd7-0c239f24ad9e,DISK]]
2024-12-09T17:05:41,981 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763926902 with entries=1, filesize=531 B; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763936939
2024-12-09T17:05:41,981 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:05:41,981 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763926902 is not closed yet, will try archiving it next time
2024-12-09T17:05:41,982 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763941982
2024-12-09T17:05:41,984 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741847_1023 (size=539)
2024-12-09T17:05:41,985 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741847_1023 (size=539)
2024-12-09T17:05:41,986 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763926902 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763926902
2024-12-09T17:05:41,993 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763936939 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763941982
2024-12-09T17:05:41,993 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39007:39007),(127.0.0.1/127.0.0.1:33915:33915)]
2024-12-09T17:05:41,993 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763936939 is not closed yet, will try archiving it next time
2024-12-09T17:05:41,993 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C41951%2C1733763851699:(num 1733763941982) roll requested
2024-12-09T17:05:41,994 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41951%2C1733763851699.1733763941993
2024-12-09T17:05:41,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741849_1025 (size=1258)
2024-12-09T17:05:41,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741849_1025 (size=1258)
2024-12-09T17:05:42,002 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763941982 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763941993
2024-12-09T17:05:42,002 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33915:33915),(127.0.0.1/127.0.0.1:39007:39007)]
2024-12-09T17:05:42,002 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763941982 is not closed yet, will try archiving it next time
2024-12-09T17:05:42,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741850_1026 (size=93)
2024-12-09T17:05:42,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741850_1026 (size=93)
2024-12-09T17:05:42,004 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699/80c69eb3c456%2C41951%2C1733763851699.1733763941982 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs/80c69eb3c456%2C41951%2C1733763851699.1733763941982
2024-12-09T17:05:44,864 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region fa1751ee8a2b3d89324db0b2e928149c, had cached 0 bytes from a total of 27710
2024-12-09T17:05:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41951 {}] regionserver.HRegion(8581): Flush requested on fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:05:54,020 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing fa1751ee8a2b3d89324db0b2e928149c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:05:54,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/220ac701c6ca46cba96f97fdabf68083 is 1080, key is row0022/info:/1733763941983/Put/seqid=0
2024-12-09T17:05:54,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741852_1028 (size=12509)
2024-12-09T17:05:54,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741852_1028 (size=12509)
2024-12-09T17:05:54,037 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/220ac701c6ca46cba96f97fdabf68083
2024-12-09T17:05:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/220ac701c6ca46cba96f97fdabf68083 as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/220ac701c6ca46cba96f97fdabf68083
2024-12-09T17:05:54,055 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/220ac701c6ca46cba96f97fdabf68083, entries=7, sequenceid=42, filesize=12.2 K
2024-12-09T17:05:54,056 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fa1751ee8a2b3d89324db0b2e928149c in 37ms, sequenceid=42, compaction requested=false
2024-12-09T17:05:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:05:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=39.3 K, sizeToCheck=16.0 K
2024-12-09T17:05:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:05:54,056 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/0c46de420a264f29a72e8596fc7ecc35 because midkey is the same as first or last row
2024-12-09T17:06:02,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-09T17:06:02,074 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-09T17:06:02,075 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x478880db to 127.0.0.1:56951
2024-12-09T17:06:02,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:02,077 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-09T17:06:02,077 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=374691138, stopped=false
2024-12-09T17:06:02,078 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=80c69eb3c456,36053,1733763850932
2024-12-09T17:06:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:02,118 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-09T17:06:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:02,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:02,119 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,41951,1733763851699' *****
2024-12-09T17:06:02,119 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:06:02,119 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:06:02,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:02,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:02,119 INFO  [RS:0;80c69eb3c456:41951 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:06:02,119 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:06:02,119 INFO  [RS:0;80c69eb3c456:41951 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:06:02,120 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(3579): Received CLOSE for fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:06:02,120 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(3579): Received CLOSE for d9eb5e5e026495891b2ae9d1a368dda7
2024-12-09T17:06:02,120 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,41951,1733763851699
2024-12-09T17:06:02,120 DEBUG [RS:0;80c69eb3c456:41951 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:02,121 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:06:02,121 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:06:02,121 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:06:02,121 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing fa1751ee8a2b3d89324db0b2e928149c, disabling compactions & flushes
2024-12-09T17:06:02,121 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:06:02,121 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:06:02,121 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:06:02,121 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c. after waiting 0 ms
2024-12-09T17:06:02,121 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-09T17:06:02,121 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:06:02,122 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:06:02,122 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1603): Online Regions={fa1751ee8a2b3d89324db0b2e928149c=TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c., 1588230740=hbase:meta,,1.1588230740, d9eb5e5e026495891b2ae9d1a368dda7=hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.}
2024-12-09T17:06:02,122 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:06:02,122 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:06:02,122 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:06:02,122 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:06:02,122 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing fa1751ee8a2b3d89324db0b2e928149c 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-09T17:06:02,122 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.81 KB heapSize=5.32 KB
2024-12-09T17:06:02,122 DEBUG [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, d9eb5e5e026495891b2ae9d1a368dda7, fa1751ee8a2b3d89324db0b2e928149c
2024-12-09T17:06:02,130 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/447b3eea9d044892b6d198e9a704fa09 is 1080, key is row0029/info:/1733763956022/Put/seqid=0
2024-12-09T17:06:02,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741853_1029 (size=8193)
2024-12-09T17:06:02,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741853_1029 (size=8193)
2024-12-09T17:06:02,137 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/447b3eea9d044892b6d198e9a704fa09
2024-12-09T17:06:02,146 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/.tmp/info/1506009e009c4475b3b1acb7e66aae7c is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c./info:regioninfo/1733763854885/Put/seqid=0
2024-12-09T17:06:02,147 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/.tmp/info/447b3eea9d044892b6d198e9a704fa09 as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/447b3eea9d044892b6d198e9a704fa09
2024-12-09T17:06:02,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741854_1030 (size=8172)
2024-12-09T17:06:02,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741854_1030 (size=8172)
2024-12-09T17:06:02,153 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.59 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/.tmp/info/1506009e009c4475b3b1acb7e66aae7c
2024-12-09T17:06:02,155 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/447b3eea9d044892b6d198e9a704fa09, entries=3, sequenceid=48, filesize=8.0 K
2024-12-09T17:06:02,156 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for fa1751ee8a2b3d89324db0b2e928149c in 34ms, sequenceid=48, compaction requested=true
2024-12-09T17:06:02,157 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/8eca4e2feac34335b96f10c26549f5da, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a29f530dcc1447d88ff8a6bd6e71eefc] to archive
2024-12-09T17:06:02,160 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-09T17:06:02,163 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032 to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/archive/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a2188f2f78c44e7184ea7e0bed76b032
2024-12-09T17:06:02,165 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/8eca4e2feac34335b96f10c26549f5da to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/archive/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/8eca4e2feac34335b96f10c26549f5da
2024-12-09T17:06:02,167 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a29f530dcc1447d88ff8a6bd6e71eefc to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/archive/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/info/a29f530dcc1447d88ff8a6bd6e71eefc
2024-12-09T17:06:02,175 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/.tmp/table/2b7ecf48aa1b456aa731e35a4aa18b4a is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733763854895/Put/seqid=0
2024-12-09T17:06:02,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741855_1031 (size=5452)
2024-12-09T17:06:02,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741855_1031 (size=5452)
2024-12-09T17:06:02,182 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=232 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/.tmp/table/2b7ecf48aa1b456aa731e35a4aa18b4a
2024-12-09T17:06:02,184 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/default/TestLogRolling-testSlowSyncLogRolling/fa1751ee8a2b3d89324db0b2e928149c/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-09T17:06:02,186 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:06:02,186 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for fa1751ee8a2b3d89324db0b2e928149c:

2024-12-09T17:06:02,187 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733763854472.fa1751ee8a2b3d89324db0b2e928149c.
2024-12-09T17:06:02,187 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d9eb5e5e026495891b2ae9d1a368dda7, disabling compactions & flushes
2024-12-09T17:06:02,187 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:06:02,187 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:06:02,187 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7. after waiting 0 ms
2024-12-09T17:06:02,187 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:06:02,187 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d9eb5e5e026495891b2ae9d1a368dda7 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-09T17:06:02,189 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/.tmp/info/1506009e009c4475b3b1acb7e66aae7c as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/info/1506009e009c4475b3b1acb7e66aae7c
2024-12-09T17:06:02,196 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/info/1506009e009c4475b3b1acb7e66aae7c, entries=20, sequenceid=14, filesize=8.0 K
2024-12-09T17:06:02,198 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/.tmp/table/2b7ecf48aa1b456aa731e35a4aa18b4a as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/table/2b7ecf48aa1b456aa731e35a4aa18b4a
2024-12-09T17:06:02,204 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/.tmp/info/4c0bd29077914abfa107d6ca6197b3bd is 45, key is default/info:d/1733763854154/Put/seqid=0
2024-12-09T17:06:02,206 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/table/2b7ecf48aa1b456aa731e35a4aa18b4a, entries=4, sequenceid=14, filesize=5.3 K
2024-12-09T17:06:02,207 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.81 KB/2882, heapSize ~5.04 KB/5160, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=14, compaction requested=false
2024-12-09T17:06:02,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741856_1032 (size=5037)
2024-12-09T17:06:02,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741856_1032 (size=5037)
2024-12-09T17:06:02,210 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/.tmp/info/4c0bd29077914abfa107d6ca6197b3bd
2024-12-09T17:06:02,214 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1
2024-12-09T17:06:02,215 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-09T17:06:02,215 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:06:02,215 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:06:02,216 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-09T17:06:02,218 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/.tmp/info/4c0bd29077914abfa107d6ca6197b3bd as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/info/4c0bd29077914abfa107d6ca6197b3bd
2024-12-09T17:06:02,226 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/info/4c0bd29077914abfa107d6ca6197b3bd, entries=2, sequenceid=6, filesize=4.9 K
2024-12-09T17:06:02,227 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for d9eb5e5e026495891b2ae9d1a368dda7 in 40ms, sequenceid=6, compaction requested=false
2024-12-09T17:06:02,231 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/data/hbase/namespace/d9eb5e5e026495891b2ae9d1a368dda7/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-09T17:06:02,232 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:06:02,232 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d9eb5e5e026495891b2ae9d1a368dda7:

2024-12-09T17:06:02,232 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733763853589.d9eb5e5e026495891b2ae9d1a368dda7.
2024-12-09T17:06:02,323 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,41951,1733763851699; all regions closed.
2024-12-09T17:06:02,324 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699
2024-12-09T17:06:02,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741834_1010 (size=4330)
2024-12-09T17:06:02,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741834_1010 (size=4330)
2024-12-09T17:06:02,329 DEBUG [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs
2024-12-09T17:06:02,329 INFO  [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C41951%2C1733763851699.meta:.meta(num 1733763853302)
2024-12-09T17:06:02,329 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/WALs/80c69eb3c456,41951,1733763851699
2024-12-09T17:06:02,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741851_1027 (size=13066)
2024-12-09T17:06:02,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741851_1027 (size=13066)
2024-12-09T17:06:02,337 DEBUG [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/oldWALs
2024-12-09T17:06:02,337 INFO  [RS:0;80c69eb3c456:41951 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C41951%2C1733763851699:(num 1733763941993)
2024-12-09T17:06:02,337 DEBUG [RS:0;80c69eb3c456:41951 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:02,337 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:06:02,338 INFO  [RS:0;80c69eb3c456:41951 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-09T17:06:02,338 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:06:02,338 INFO  [RS:0;80c69eb3c456:41951 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41951
2024-12-09T17:06:02,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:06:02,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,41951,1733763851699
2024-12-09T17:06:02,360 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,41951,1733763851699]
2024-12-09T17:06:02,360 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,41951,1733763851699; numProcessing=1
2024-12-09T17:06:02,368 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,41951,1733763851699 already deleted, retry=false
2024-12-09T17:06:02,368 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,41951,1733763851699 expired; onlineServers=0
2024-12-09T17:06:02,368 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,36053,1733763850932' *****
2024-12-09T17:06:02,368 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-09T17:06:02,369 DEBUG [M:0;80c69eb3c456:36053 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d426fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:06:02,369 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,36053,1733763850932
2024-12-09T17:06:02,369 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,36053,1733763850932; all regions closed.
2024-12-09T17:06:02,369 DEBUG [M:0;80c69eb3c456:36053 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:02,369 DEBUG [M:0;80c69eb3c456:36053 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-09T17:06:02,369 DEBUG [M:0;80c69eb3c456:36053 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-09T17:06:02,369 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-09T17:06:02,369 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733763852572 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733763852572,5,FailOnTimeoutGroup]
2024-12-09T17:06:02,369 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733763852572 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733763852572,5,FailOnTimeoutGroup]
2024-12-09T17:06:02,369 INFO  [M:0;80c69eb3c456:36053 {}] hbase.ChoreService(370): Chore service for: master/80c69eb3c456:0 had [] on shutdown
2024-12-09T17:06:02,370 DEBUG [M:0;80c69eb3c456:36053 {}] master.HMaster(1733): Stopping service threads
2024-12-09T17:06:02,370 INFO  [M:0;80c69eb3c456:36053 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-09T17:06:02,370 INFO  [M:0;80c69eb3c456:36053 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-09T17:06:02,370 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-09T17:06:02,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-09T17:06:02,376 DEBUG [M:0;80c69eb3c456:36053 {}] zookeeper.ZKUtil(347): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-09T17:06:02,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:02,376 WARN  [M:0;80c69eb3c456:36053 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-09T17:06:02,377 INFO  [M:0;80c69eb3c456:36053 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-09T17:06:02,377 INFO  [M:0;80c69eb3c456:36053 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-09T17:06:02,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:06:02,377 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:06:02,377 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:02,377 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:02,377 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:06:02,377 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:02,378 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.22 KB heapSize=50.15 KB
2024-12-09T17:06:02,395 DEBUG [M:0;80c69eb3c456:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5e2cc16dc2c74e1fa0ec2b49069140e4 is 82, key is hbase:meta,,1/info:regioninfo/1733763853409/Put/seqid=0
2024-12-09T17:06:02,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741857_1033 (size=5672)
2024-12-09T17:06:02,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741857_1033 (size=5672)
2024-12-09T17:06:02,403 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5e2cc16dc2c74e1fa0ec2b49069140e4
2024-12-09T17:06:02,424 DEBUG [M:0;80c69eb3c456:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1ea87e75d73498d910af4ede4c2407c is 767, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733763854941/Put/seqid=0
2024-12-09T17:06:02,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741858_1034 (size=6427)
2024-12-09T17:06:02,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741858_1034 (size=6427)
2024-12-09T17:06:02,430 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.62 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1ea87e75d73498d910af4ede4c2407c
2024-12-09T17:06:02,436 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e1ea87e75d73498d910af4ede4c2407c
2024-12-09T17:06:02,451 DEBUG [M:0;80c69eb3c456:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ea1016783b640028e5e66b2ebfa034c is 69, key is 80c69eb3c456,41951,1733763851699/rs:state/1733763852678/Put/seqid=0
2024-12-09T17:06:02,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741859_1035 (size=5156)
2024-12-09T17:06:02,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741859_1035 (size=5156)
2024-12-09T17:06:02,456 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ea1016783b640028e5e66b2ebfa034c
2024-12-09T17:06:02,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:02,460 INFO  [RS:0;80c69eb3c456:41951 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,41951,1733763851699; zookeeper connection closed.
2024-12-09T17:06:02,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41951-0x1000bbffc400001, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:02,460 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f2f1809 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f2f1809
2024-12-09T17:06:02,460 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-09T17:06:02,476 DEBUG [M:0;80c69eb3c456:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5c04621f4c1e4e069647be56fc61e848 is 52, key is load_balancer_on/state:d/1733763854456/Put/seqid=0
2024-12-09T17:06:02,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741860_1036 (size=5056)
2024-12-09T17:06:02,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741860_1036 (size=5056)
2024-12-09T17:06:02,482 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5c04621f4c1e4e069647be56fc61e848
2024-12-09T17:06:02,488 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5e2cc16dc2c74e1fa0ec2b49069140e4 as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5e2cc16dc2c74e1fa0ec2b49069140e4
2024-12-09T17:06:02,495 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5e2cc16dc2c74e1fa0ec2b49069140e4, entries=8, sequenceid=104, filesize=5.5 K
2024-12-09T17:06:02,496 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1ea87e75d73498d910af4ede4c2407c as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e1ea87e75d73498d910af4ede4c2407c
2024-12-09T17:06:02,502 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e1ea87e75d73498d910af4ede4c2407c
2024-12-09T17:06:02,502 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e1ea87e75d73498d910af4ede4c2407c, entries=11, sequenceid=104, filesize=6.3 K
2024-12-09T17:06:02,503 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ea1016783b640028e5e66b2ebfa034c as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ea1016783b640028e5e66b2ebfa034c
2024-12-09T17:06:02,510 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ea1016783b640028e5e66b2ebfa034c, entries=1, sequenceid=104, filesize=5.0 K
2024-12-09T17:06:02,511 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5c04621f4c1e4e069647be56fc61e848 as hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5c04621f4c1e4e069647be56fc61e848
2024-12-09T17:06:02,518 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5c04621f4c1e4e069647be56fc61e848, entries=1, sequenceid=104, filesize=4.9 K
2024-12-09T17:06:02,519 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.22 KB/41185, heapSize ~50.09 KB/51288, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 142ms, sequenceid=104, compaction requested=false
2024-12-09T17:06:02,520 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:02,520 DEBUG [M:0;80c69eb3c456:36053 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:06:02,521 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/MasterData/WALs/80c69eb3c456,36053,1733763850932
2024-12-09T17:06:02,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741830_1006 (size=48486)
2024-12-09T17:06:02,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44791 is added to blk_1073741830_1006 (size=48486)
2024-12-09T17:06:02,523 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:06:02,523 INFO  [M:0;80c69eb3c456:36053 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-09T17:06:02,523 INFO  [M:0;80c69eb3c456:36053 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36053
2024-12-09T17:06:02,532 DEBUG [M:0;80c69eb3c456:36053 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/80c69eb3c456,36053,1733763850932 already deleted, retry=false
2024-12-09T17:06:02,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:02,641 INFO  [M:0;80c69eb3c456:36053 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,36053,1733763850932; zookeeper connection closed.
2024-12-09T17:06:02,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1000bbffc400000, quorum=127.0.0.1:56951, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:02,650 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@163cfad6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:02,653 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:02,653 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:02,653 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:02,653 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:02,656 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:02,656 WARN  [BP-2139196168-172.17.0.2-1733763847296 heartbeating to localhost/127.0.0.1:44739 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:02,657 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:02,657 WARN  [BP-2139196168-172.17.0.2-1733763847296 heartbeating to localhost/127.0.0.1:44739 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2139196168-172.17.0.2-1733763847296 (Datanode Uuid 16401565-5422-4d09-b9b9-c82058ffbcc2) service to localhost/127.0.0.1:44739
2024-12-09T17:06:02,658 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data3/current/BP-2139196168-172.17.0.2-1733763847296 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:02,659 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data4/current/BP-2139196168-172.17.0.2-1733763847296 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:02,659 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:02,662 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6aad8790{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:02,663 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:02,663 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:02,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:02,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:02,665 WARN  [BP-2139196168-172.17.0.2-1733763847296 heartbeating to localhost/127.0.0.1:44739 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:02,665 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:02,665 WARN  [BP-2139196168-172.17.0.2-1733763847296 heartbeating to localhost/127.0.0.1:44739 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2139196168-172.17.0.2-1733763847296 (Datanode Uuid d528a27f-b13f-4ff6-8c0c-00bd168fc3b1) service to localhost/127.0.0.1:44739
2024-12-09T17:06:02,665 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:02,665 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data1/current/BP-2139196168-172.17.0.2-1733763847296 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:02,666 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/cluster_2bc35f96-81e6-54ae-d925-58e874c7d2d6/dfs/data/data2/current/BP-2139196168-172.17.0.2-1733763847296 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:02,666 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:02,675 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5682c4d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:06:02,676 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:02,676 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:02,676 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:02,676 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:02,684 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-09T17:06:02,716 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-09T17:06:02,722 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=65 (was 12)
Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@170a868d
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44739
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:44739
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44739
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44739
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:44739 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:44739 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/80c69eb3c456:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: RS-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/80c69eb3c456:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:44739 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RS-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44739
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/80c69eb3c456:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/80c69eb3c456:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=403 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=77 (was 323), ProcessCount=11 (was 11), AvailableMemoryMB=5348 (was 5882)
2024-12-09T17:06:02,727 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=66, OpenFileDescriptor=403, MaxFileDescriptor=1048576, SystemLoadAverage=77, ProcessCount=11, AvailableMemoryMB=5348
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.log.dir so I do NOT create it in target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/495a79ab-62f5-d55f-e19d-e50b4ae8ec3d/hadoop.tmp.dir so I do NOT create it in target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9, deleteOnExit=true
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/test.cache.data in system properties and HBase conf
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.tmp.dir in system properties and HBase conf
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir in system properties and HBase conf
2024-12-09T17:06:02,728 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-09T17:06:02,729 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:06:02,729 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-09T17:06:02,730 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/nfs.dump.dir in system properties and HBase conf
2024-12-09T17:06:02,730 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir in system properties and HBase conf
2024-12-09T17:06:02,730 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:06:02,730 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-09T17:06:02,730 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-09T17:06:02,742 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:06:02,760 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:06:03,028 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:03,034 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:03,035 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:03,035 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:03,035 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:06:03,036 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:03,036 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a43de3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:03,036 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56fd08b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:03,127 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@381717fc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-42885-hadoop-hdfs-3_4_1-tests_jar-_-any-3525869518050005964/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:06:03,128 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43db7010{HTTP/1.1, (http/1.1)}{localhost:42885}
2024-12-09T17:06:03,128 INFO  [Time-limited test {}] server.Server(415): Started @117548ms
2024-12-09T17:06:03,139 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:06:03,330 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:03,333 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:03,336 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:03,336 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:03,336 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:06:03,337 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b1c985{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:03,338 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22b28729{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:03,429 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c96e968{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-45999-hadoop-hdfs-3_4_1-tests_jar-_-any-4222469871319954292/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:03,430 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21edda6b{HTTP/1.1, (http/1.1)}{localhost:45999}
2024-12-09T17:06:03,430 INFO  [Time-limited test {}] server.Server(415): Started @117849ms
2024-12-09T17:06:03,431 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:06:03,473 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:03,478 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:03,480 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:03,480 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:03,480 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:06:03,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63d13cc5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:03,481 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62cd5620{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:03,570 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38f56d13{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-38267-hadoop-hdfs-3_4_1-tests_jar-_-any-7551091229058450046/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:03,570 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d72d7be{HTTP/1.1, (http/1.1)}{localhost:38267}
2024-12-09T17:06:03,570 INFO  [Time-limited test {}] server.Server(415): Started @117990ms
2024-12-09T17:06:03,571 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:06:04,107 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data1/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:04,107 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data2/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:04,127 WARN  [Thread-428 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:06:04,129 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3eb48a50b94cddaf with lease ID 0xaab4f63442d14829: Processing first storage report for DS-be35cfe3-5010-4a88-96aa-c63a027da5a2 from datanode DatanodeRegistration(127.0.0.1:34177, datanodeUuid=71d8b779-e8b0-4886-99d2-88560251626f, infoPort=34629, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:04,129 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3eb48a50b94cddaf with lease ID 0xaab4f63442d14829: from storage DS-be35cfe3-5010-4a88-96aa-c63a027da5a2 node DatanodeRegistration(127.0.0.1:34177, datanodeUuid=71d8b779-e8b0-4886-99d2-88560251626f, infoPort=34629, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:04,129 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3eb48a50b94cddaf with lease ID 0xaab4f63442d14829: Processing first storage report for DS-ec78078a-3a5e-4bd6-856c-5e82892fb1cd from datanode DatanodeRegistration(127.0.0.1:34177, datanodeUuid=71d8b779-e8b0-4886-99d2-88560251626f, infoPort=34629, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:04,130 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3eb48a50b94cddaf with lease ID 0xaab4f63442d14829: from storage DS-ec78078a-3a5e-4bd6-856c-5e82892fb1cd node DatanodeRegistration(127.0.0.1:34177, datanodeUuid=71d8b779-e8b0-4886-99d2-88560251626f, infoPort=34629, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:04,264 WARN  [Thread-475 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data3/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:04,264 WARN  [Thread-476 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data4/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:04,279 WARN  [Thread-451 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:06:04,281 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11395f2f19695e09 with lease ID 0xaab4f63442d1482a: Processing first storage report for DS-bd814197-03fc-4480-b40d-da61c2c800df from datanode DatanodeRegistration(127.0.0.1:43139, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=37647, infoSecurePort=0, ipcPort=44451, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:04,281 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11395f2f19695e09 with lease ID 0xaab4f63442d1482a: from storage DS-bd814197-03fc-4480-b40d-da61c2c800df node DatanodeRegistration(127.0.0.1:43139, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=37647, infoSecurePort=0, ipcPort=44451, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:04,281 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11395f2f19695e09 with lease ID 0xaab4f63442d1482a: Processing first storage report for DS-868b3bc6-dd21-46a9-92b9-fdebd76e0ee7 from datanode DatanodeRegistration(127.0.0.1:43139, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=37647, infoSecurePort=0, ipcPort=44451, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:04,281 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11395f2f19695e09 with lease ID 0xaab4f63442d1482a: from storage DS-868b3bc6-dd21-46a9-92b9-fdebd76e0ee7 node DatanodeRegistration(127.0.0.1:43139, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=37647, infoSecurePort=0, ipcPort=44451, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:04,305 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce
2024-12-09T17:06:04,309 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/zookeeper_0, clientPort=60386, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-09T17:06:04,311 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60386
2024-12-09T17:06:04,311 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,313 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:06:04,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:06:04,327 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3 with version=8
2024-12-09T17:06:04,327 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase-staging
2024-12-09T17:06:04,329 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:06:04,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:04,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:04,329 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:06:04,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:04,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:06:04,330 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:06:04,330 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:06:04,330 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44805
2024-12-09T17:06:04,331 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,332 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,335 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:44805 connecting to ZooKeeper ensemble=127.0.0.1:60386
2024-12-09T17:06:04,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448050x0, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:06:04,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44805-0x1000bc1ba140000 connected
2024-12-09T17:06:04,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:06:04,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:04,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:06:04,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44805
2024-12-09T17:06:04,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44805
2024-12-09T17:06:04,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44805
2024-12-09T17:06:04,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44805
2024-12-09T17:06:04,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44805
2024-12-09T17:06:04,459 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3, hbase.cluster.distributed=false
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:06:04,477 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:06:04,479 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40321
2024-12-09T17:06:04,480 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:06:04,480 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:06:04,481 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,483 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,485 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:40321 connecting to ZooKeeper ensemble=127.0.0.1:60386
2024-12-09T17:06:04,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403210x0, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:06:04,493 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:403210x0, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:06:04,493 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40321-0x1000bc1ba140001 connected
2024-12-09T17:06:04,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:04,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:06:04,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40321
2024-12-09T17:06:04,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40321
2024-12-09T17:06:04,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40321
2024-12-09T17:06:04,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40321
2024-12-09T17:06:04,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40321
2024-12-09T17:06:04,501 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:06:04,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:06:04,510 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,512 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80c69eb3c456:44805
2024-12-09T17:06:04,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:06:04,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:06:04,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,518 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:06:04,519 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80c69eb3c456,44805,1733763964328 from backup master directory
2024-12-09T17:06:04,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:06:04,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:06:04,526 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:06:04,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:06:04,526 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:06:04,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:06:04,539 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/hbase.id with ID: fb514127-b151-4511-b566-77d1420ef4c2
2024-12-09T17:06:04,552 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:04,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:06:04,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:06:04,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:06:04,570 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-09T17:06:04,570 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:06:04,581 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:06:04,581 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:06:04,582 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store
2024-12-09T17:06:04,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:06:04,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:06:04,591 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:04,591 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:06:04,591 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:04,591 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:04,591 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:06:04,591 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:04,591 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:04,591 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:06:04,592 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/.initializing
2024-12-09T17:06:04,592 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,595 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C44805%2C1733763964328, suffix=, logDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328, archiveDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/oldWALs, maxLogs=10
2024-12-09T17:06:04,596 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C44805%2C1733763964328.1733763964596
2024-12-09T17:06:04,602 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596
2024-12-09T17:06:04,602 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34629:34629),(127.0.0.1/127.0.0.1:37647:37647)]
2024-12-09T17:06:04,603 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:06:04,603 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:04,603 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:06:04,603 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:06:04,605 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:06:04,606 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-09T17:06:04,606 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,607 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:04,607 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:06:04,609 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-09T17:06:04,609 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,609 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:06:04,609 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:06:04,611 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-09T17:06:04,611 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,612 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:06:04,612 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:06:04,615 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-09T17:06:04,616 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,616 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:06:04,617 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:06:04,618 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:06:04,620 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-09T17:06:04,621 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:06:04,624 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:06:04,624 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=797420, jitterRate=0.013972729444503784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-09T17:06:04,625 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:06:04,625 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-09T17:06:04,630 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc8c862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:06:04,631 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-09T17:06:04,631 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-09T17:06:04,631 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-09T17:06:04,631 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-09T17:06:04,632 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-09T17:06:04,633 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-09T17:06:04,633 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-09T17:06:04,635 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-09T17:06:04,636 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-09T17:06:04,643 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-09T17:06:04,643 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-09T17:06:04,644 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-09T17:06:04,651 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-09T17:06:04,651 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-09T17:06:04,653 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-09T17:06:04,659 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-09T17:06:04,660 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-09T17:06:04,668 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-09T17:06:04,671 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-09T17:06:04,676 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-09T17:06:04,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:04,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:04,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,686 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=80c69eb3c456,44805,1733763964328, sessionid=0x1000bc1ba140000, setting cluster-up flag (Was=false)
2024-12-09T17:06:04,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,726 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-09T17:06:04,729 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:04,776 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-09T17:06:04,778 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,44805,1733763964328
2024-12-09T17:06:04,781 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-09T17:06:04,781 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-09T17:06:04,782 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-09T17:06:04,782 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80c69eb3c456,44805,1733763964328
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-09T17:06:04,782 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:06:04,782 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:06:04,782 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:06:04,782 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:06:04,783 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80c69eb3c456:0, corePoolSize=10, maxPoolSize=10
2024-12-09T17:06:04,783 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,783 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:06:04,783 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733763994787
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-09T17:06:04,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,788 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:06:04,788 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-09T17:06:04,788 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-09T17:06:04,788 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-09T17:06:04,788 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-09T17:06:04,788 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-09T17:06:04,788 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-09T17:06:04,789 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733763964788,5,FailOnTimeoutGroup]
2024-12-09T17:06:04,789 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733763964789,5,FailOnTimeoutGroup]
2024-12-09T17:06:04,789 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,789 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-09T17:06:04,789 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,789 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,789 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,789 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:06:04,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:06:04,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:06:04,802 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-09T17:06:04,803 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3
2024-12-09T17:06:04,814 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80c69eb3c456:40321
2024-12-09T17:06:04,815 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1008): ClusterId : fb514127-b151-4511-b566-77d1420ef4c2
2024-12-09T17:06:04,815 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:06:04,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:06:04,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:06:04,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:04,823 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:06:04,825 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:06:04,825 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:06:04,825 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:06:04,825 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,826 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:04,826 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:06:04,828 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:06:04,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,828 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:04,828 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:06:04,830 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:06:04,830 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:04,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:04,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/meta/1588230740
2024-12-09T17:06:04,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/meta/1588230740
2024-12-09T17:06:04,834 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:06:04,835 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:06:04,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:06:04,835 DEBUG [RS:0;80c69eb3c456:40321 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254a49ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:06:04,836 DEBUG [RS:0;80c69eb3c456:40321 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536fa812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:06:04,836 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:06:04,836 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:06:04,836 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:06:04,837 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,44805,1733763964328 with isa=80c69eb3c456/172.17.0.2:40321, startcode=1733763964476
2024-12-09T17:06:04,837 DEBUG [RS:0;80c69eb3c456:40321 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:06:04,838 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:06:04,839 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741809, jitterRate=-0.05674216151237488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:06:04,840 INFO  [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40913, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:06:04,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:06:04,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:06:04,840 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:06:04,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:06:04,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:06:04,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:06:04,840 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44805 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:04,840 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44805 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:04,841 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:06:04,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:06:04,842 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3
2024-12-09T17:06:04,842 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:06:04,842 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:34441
2024-12-09T17:06:04,842 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-09T17:06:04,842 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:06:04,842 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-09T17:06:04,844 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-09T17:06:04,845 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-09T17:06:04,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:06:04,852 DEBUG [RS:0;80c69eb3c456:40321 {}] zookeeper.ZKUtil(111): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,40321,1733763964476
2024-12-09T17:06:04,852 WARN  [RS:0;80c69eb3c456:40321 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:06:04,852 INFO  [RS:0;80c69eb3c456:40321 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:06:04,852 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476
2024-12-09T17:06:04,852 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,40321,1733763964476]
2024-12-09T17:06:04,858 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:06:04,858 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:06:04,860 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:06:04,861 INFO  [RS:0;80c69eb3c456:40321 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:06:04,861 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,861 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:06:04,862 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,862 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,862 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,862 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,862 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:06:04,863 DEBUG [RS:0;80c69eb3c456:40321 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:06:04,864 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,864 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,864 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,864 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,864 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,40321,1733763964476-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:06:04,882 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:06:04,883 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,40321,1733763964476-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:04,896 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.Replication(204): 80c69eb3c456,40321,1733763964476 started
2024-12-09T17:06:04,896 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,40321,1733763964476, RpcServer on 80c69eb3c456/172.17.0.2:40321, sessionid=0x1000bc1ba140001
2024-12-09T17:06:04,897 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:06:04,897 DEBUG [RS:0;80c69eb3c456:40321 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:04,897 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,40321,1733763964476'
2024-12-09T17:06:04,897 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:06:04,897 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:06:04,898 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:06:04,898 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:06:04,898 DEBUG [RS:0;80c69eb3c456:40321 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:04,898 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,40321,1733763964476'
2024-12-09T17:06:04,898 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:06:04,899 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:06:04,899 DEBUG [RS:0;80c69eb3c456:40321 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:06:04,899 INFO  [RS:0;80c69eb3c456:40321 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:06:04,899 INFO  [RS:0;80c69eb3c456:40321 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:06:04,995 WARN  [80c69eb3c456:44805 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-09T17:06:05,003 INFO  [RS:0;80c69eb3c456:40321 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C40321%2C1733763964476, suffix=, logDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476, archiveDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs, maxLogs=32
2024-12-09T17:06:05,008 INFO  [RS:0;80c69eb3c456:40321 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.1733763965007
2024-12-09T17:06:05,017 INFO  [RS:0;80c69eb3c456:40321 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007
2024-12-09T17:06:05,018 DEBUG [RS:0;80c69eb3c456:40321 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37647:37647),(127.0.0.1/127.0.0.1:34629:34629)]
2024-12-09T17:06:05,246 DEBUG [80c69eb3c456:44805 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-09T17:06:05,246 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:05,249 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,40321,1733763964476, state=OPENING
2024-12-09T17:06:05,291 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-09T17:06:05,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:05,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:05,304 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=80c69eb3c456,40321,1733763964476}]
2024-12-09T17:06:05,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:06:05,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:06:05,460 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:05,460 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-09T17:06:05,463 INFO  [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37204, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-09T17:06:05,469 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-09T17:06:05,470 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:06:05,473 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C40321%2C1733763964476.meta, suffix=.meta, logDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476, archiveDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs, maxLogs=32
2024-12-09T17:06:05,477 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
2024-12-09T17:06:05,484 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
2024-12-09T17:06:05,484 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37647:37647),(127.0.0.1/127.0.0.1:34629:34629)]
2024-12-09T17:06:05,484 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:06:05,485 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-09T17:06:05,485 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-09T17:06:05,485 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-09T17:06:05,485 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-09T17:06:05,485 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:05,485 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-09T17:06:05,485 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-09T17:06:05,487 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:06:05,488 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:06:05,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:05,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:05,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:06:05,489 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:06:05,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:05,490 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:05,490 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:06:05,491 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:06:05,491 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:05,491 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:06:05,492 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/meta/1588230740
2024-12-09T17:06:05,493 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/meta/1588230740
2024-12-09T17:06:05,495 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:06:05,496 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:06:05,497 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826757, jitterRate=0.05127696692943573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:06:05,498 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:06:05,499 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733763965460
2024-12-09T17:06:05,501 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-09T17:06:05,501 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-09T17:06:05,502 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:05,503 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,40321,1733763964476, state=OPEN
2024-12-09T17:06:05,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:06:05,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:06:05,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:06:05,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:06:05,534 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-09T17:06:05,535 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=80c69eb3c456,40321,1733763964476 in 227 msec
2024-12-09T17:06:05,538 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-09T17:06:05,538 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 692 msec
2024-12-09T17:06:05,540 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 758 msec
2024-12-09T17:06:05,540 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733763965540, completionTime=-1
2024-12-09T17:06:05,540 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-09T17:06:05,541 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-09T17:06:05,541 DEBUG [hconnection-0x397c1e42-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:06:05,543 INFO  [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37212, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:06:05,544 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-09T17:06:05,544 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733764025544
2024-12-09T17:06:05,544 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733764085544
2024-12-09T17:06:05,544 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-09T17:06:05,568 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44805,1733763964328-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:05,568 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44805,1733763964328-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:05,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44805,1733763964328-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:05,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80c69eb3c456:44805, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:05,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:05,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-09T17:06:05,569 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:06:05,572 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-09T17:06:05,572 DEBUG [master/80c69eb3c456:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-09T17:06:05,574 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:06:05,575 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:05,577 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:06:05,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:06:05,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:06:05,590 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 218a4c3d52fad4eee63ea39e5e84c0ca, NAME => 'hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3
2024-12-09T17:06:05,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:06:05,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:06:05,601 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:05,601 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 218a4c3d52fad4eee63ea39e5e84c0ca, disabling compactions & flushes
2024-12-09T17:06:05,601 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,601 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,601 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca. after waiting 0 ms
2024-12-09T17:06:05,601 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,601 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,601 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 218a4c3d52fad4eee63ea39e5e84c0ca:

2024-12-09T17:06:05,602 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:06:05,602 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733763965602"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733763965602"}]},"ts":"1733763965602"}
2024-12-09T17:06:05,604 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:06:05,605 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:06:05,605 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763965605"}]},"ts":"1733763965605"}
2024-12-09T17:06:05,607 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-09T17:06:05,626 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=218a4c3d52fad4eee63ea39e5e84c0ca, ASSIGN}]
2024-12-09T17:06:05,628 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=218a4c3d52fad4eee63ea39e5e84c0ca, ASSIGN
2024-12-09T17:06:05,630 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=218a4c3d52fad4eee63ea39e5e84c0ca, ASSIGN; state=OFFLINE, location=80c69eb3c456,40321,1733763964476; forceNewPlan=false, retain=false
2024-12-09T17:06:05,781 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=218a4c3d52fad4eee63ea39e5e84c0ca, regionState=OPENING, regionLocation=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:05,788 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 218a4c3d52fad4eee63ea39e5e84c0ca, server=80c69eb3c456,40321,1733763964476}]
2024-12-09T17:06:05,943 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:05,952 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 218a4c3d52fad4eee63ea39e5e84c0ca, NAME => 'hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:06:05,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:05,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:05,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:05,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:05,956 INFO  [StoreOpener-218a4c3d52fad4eee63ea39e5e84c0ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 218a4c3d52fad4eee63ea39e5e84c0ca 
2024-12-09T17:06:05,959 INFO  [StoreOpener-218a4c3d52fad4eee63ea39e5e84c0ca-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 218a4c3d52fad4eee63ea39e5e84c0ca columnFamilyName info
2024-12-09T17:06:05,959 DEBUG [StoreOpener-218a4c3d52fad4eee63ea39e5e84c0ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:05,959 INFO  [StoreOpener-218a4c3d52fad4eee63ea39e5e84c0ca-1 {}] regionserver.HStore(327): Store=218a4c3d52fad4eee63ea39e5e84c0ca/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:06:05,961 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/namespace/218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:05,961 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/namespace/218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:05,964 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:05,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/hbase/namespace/218a4c3d52fad4eee63ea39e5e84c0ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:06:05,968 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 218a4c3d52fad4eee63ea39e5e84c0ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881843, jitterRate=0.12132230401039124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:06:05,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 218a4c3d52fad4eee63ea39e5e84c0ca:

2024-12-09T17:06:05,970 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca., pid=6, masterSystemTime=1733763965943
2024-12-09T17:06:05,973 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,973 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:05,974 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=218a4c3d52fad4eee63ea39e5e84c0ca, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:05,981 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-09T17:06:05,981 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 218a4c3d52fad4eee63ea39e5e84c0ca, server=80c69eb3c456,40321,1733763964476 in 189 msec
2024-12-09T17:06:05,985 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-09T17:06:05,985 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=218a4c3d52fad4eee63ea39e5e84c0ca, ASSIGN in 355 msec
2024-12-09T17:06:05,987 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:06:05,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763965987"}]},"ts":"1733763965987"}
2024-12-09T17:06:05,990 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-09T17:06:06,067 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:06:06,070 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 498 msec
2024-12-09T17:06:06,074 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-09T17:06:06,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:06:06,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:06,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:06,120 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-09T17:06:06,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:06:06,146 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 26 msec
2024-12-09T17:06:06,153 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-09T17:06:06,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:06:06,179 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 25 msec
2024-12-09T17:06:06,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-09T17:06:06,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.692sec
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44805,1733763964328-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:06:06,218 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44805,1733763964328-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-09T17:06:06,220 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-09T17:06:06,221 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-09T17:06:06,221 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44805,1733763964328-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,304 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ad056f1 to 127.0.0.1:60386 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2117a57e
2024-12-09T17:06:06,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad53901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:06:06,320 DEBUG [hconnection-0x25988dd3-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:06:06,323 INFO  [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37222, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:06:06,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=80c69eb3c456,44805,1733763964328
2024-12-09T17:06:06,327 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:06,331 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-09T17:06:06,349 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:06:06,349 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:06,349 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:06,349 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:06:06,349 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:06:06,350 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:06:06,350 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:06:06,350 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:06:06,350 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43075
2024-12-09T17:06:06,351 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:06:06,352 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:06:06,352 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:06,354 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:06:06,356 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:43075 connecting to ZooKeeper ensemble=127.0.0.1:60386
2024-12-09T17:06:06,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430750x0, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:06:06,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:06:06,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43075-0x1000bc1ba140003 connected
2024-12-09T17:06:06,367 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-09T17:06:06,368 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:06:06,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43075
2024-12-09T17:06:06,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43075
2024-12-09T17:06:06,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43075
2024-12-09T17:06:06,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43075
2024-12-09T17:06:06,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43075
2024-12-09T17:06:06,371 DEBUG [pool-282-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-09T17:06:06,385 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;80c69eb3c456:43075
2024-12-09T17:06:06,386 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1008): ClusterId : fb514127-b151-4511-b566-77d1420ef4c2
2024-12-09T17:06:06,386 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:06:06,394 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:06:06,394 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:06:06,402 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:06:06,403 DEBUG [RS:1;80c69eb3c456:43075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e73695c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:06:06,403 DEBUG [RS:1;80c69eb3c456:43075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f19842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:06:06,403 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:06:06,403 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:06:06,403 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:06:06,404 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,44805,1733763964328 with isa=80c69eb3c456/172.17.0.2:43075, startcode=1733763966349
2024-12-09T17:06:06,404 DEBUG [RS:1;80c69eb3c456:43075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:06:06,406 INFO  [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49845, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:06:06,406 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44805 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,43075,1733763966349
2024-12-09T17:06:06,407 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44805 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,43075,1733763966349
2024-12-09T17:06:06,409 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3
2024-12-09T17:06:06,409 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:34441
2024-12-09T17:06:06,409 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:06:06,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:06:06,419 DEBUG [RS:1;80c69eb3c456:43075 {}] zookeeper.ZKUtil(111): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,43075,1733763966349
2024-12-09T17:06:06,419 WARN  [RS:1;80c69eb3c456:43075 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:06:06,419 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,43075,1733763966349]
2024-12-09T17:06:06,419 INFO  [RS:1;80c69eb3c456:43075 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:06:06,419 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,43075,1733763966349
2024-12-09T17:06:06,426 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:06:06,426 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:06:06,430 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:06:06,430 INFO  [RS:1;80c69eb3c456:43075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:06:06,430 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,431 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:06:06,432 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,432 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,433 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,433 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,433 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:06:06,433 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:06:06,433 DEBUG [RS:1;80c69eb3c456:43075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:06:06,433 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,434 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,434 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,434 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,434 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,43075,1733763966349-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:06:06,449 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:06:06,449 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,43075,1733763966349-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:06:06,461 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.Replication(204): 80c69eb3c456,43075,1733763966349 started
2024-12-09T17:06:06,461 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,43075,1733763966349, RpcServer on 80c69eb3c456/172.17.0.2:43075, sessionid=0x1000bc1ba140003
2024-12-09T17:06:06,461 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:06:06,461 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(3355): Started new server=Thread[RS:1;80c69eb3c456:43075,5,FailOnTimeoutGroup]
2024-12-09T17:06:06,461 DEBUG [RS:1;80c69eb3c456:43075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,43075,1733763966349
2024-12-09T17:06:06,461 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,43075,1733763966349'
2024-12-09T17:06:06,461 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:06:06,461 INFO  [Time-limited test {}] wal.TestLogRolling(191): Replication=2
2024-12-09T17:06:06,462 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:06:06,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-09T17:06:06,462 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:06:06,462 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:06:06,462 DEBUG [RS:1;80c69eb3c456:43075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,43075,1733763966349
2024-12-09T17:06:06,462 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,43075,1733763966349'
2024-12-09T17:06:06,462 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:06:06,463 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:06:06,463 DEBUG [RS:1;80c69eb3c456:43075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:06:06,463 INFO  [RS:1;80c69eb3c456:43075 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:06:06,463 INFO  [RS:1;80c69eb3c456:43075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:06:06,464 INFO  [RS-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60036, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-09T17:06:06,465 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-09T17:06:06,465 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-09T17:06:06,465 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:06:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-09T17:06:06,468 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:06:06,468 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:06,468 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 9
2024-12-09T17:06:06,469 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:06:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:06:06,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741837_1013 (size=393)
2024-12-09T17:06:06,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741837_1013 (size=393)
2024-12-09T17:06:06,478 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 500360dd67d4a36f45916b1c7a30f4aa, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3
2024-12-09T17:06:06,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43139 is added to blk_1073741838_1014 (size=76)
2024-12-09T17:06:06,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34177 is added to blk_1073741838_1014 (size=76)
2024-12-09T17:06:06,490 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:06,490 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1681): Closing 500360dd67d4a36f45916b1c7a30f4aa, disabling compactions & flushes
2024-12-09T17:06:06,490 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,490 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,490 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa. after waiting 0 ms
2024-12-09T17:06:06,490 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,490 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,490 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1635): Region close journal for 500360dd67d4a36f45916b1c7a30f4aa:

2024-12-09T17:06:06,492 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:06:06,492 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733763966492"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733763966492"}]},"ts":"1733763966492"}
2024-12-09T17:06:06,494 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:06:06,495 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:06:06,495 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763966495"}]},"ts":"1733763966495"}
2024-12-09T17:06:06,497 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-09T17:06:06,516 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=500360dd67d4a36f45916b1c7a30f4aa, ASSIGN}]
2024-12-09T17:06:06,518 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=500360dd67d4a36f45916b1c7a30f4aa, ASSIGN
2024-12-09T17:06:06,519 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=500360dd67d4a36f45916b1c7a30f4aa, ASSIGN; state=OFFLINE, location=80c69eb3c456,40321,1733763964476; forceNewPlan=false, retain=false
2024-12-09T17:06:06,587 INFO  [RS:1;80c69eb3c456:43075 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C43075%2C1733763966349, suffix=, logDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,43075,1733763966349, archiveDir=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs, maxLogs=32
2024-12-09T17:06:06,588 INFO  [RS:1;80c69eb3c456:43075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C43075%2C1733763966349.1733763966588
2024-12-09T17:06:06,594 INFO  [RS:1;80c69eb3c456:43075 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,43075,1733763966349/80c69eb3c456%2C43075%2C1733763966349.1733763966588
2024-12-09T17:06:06,594 DEBUG [RS:1;80c69eb3c456:43075 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37647:37647),(127.0.0.1/127.0.0.1:34629:34629)]
2024-12-09T17:06:06,671 INFO  [80c69eb3c456:44805 {}] balancer.BaseLoadBalancer(546): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-09T17:06:06,672 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=500360dd67d4a36f45916b1c7a30f4aa, regionState=OPENING, regionLocation=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:06,676 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 500360dd67d4a36f45916b1c7a30f4aa, server=80c69eb3c456,40321,1733763964476}]
2024-12-09T17:06:06,831 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:06,840 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,840 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 500360dd67d4a36f45916b1c7a30f4aa, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:06:06,841 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:06,841 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:06:06,841 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:06,842 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:06,844 INFO  [StoreOpener-500360dd67d4a36f45916b1c7a30f4aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 500360dd67d4a36f45916b1c7a30f4aa 
2024-12-09T17:06:06,846 INFO  [StoreOpener-500360dd67d4a36f45916b1c7a30f4aa-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 500360dd67d4a36f45916b1c7a30f4aa columnFamilyName info
2024-12-09T17:06:06,846 DEBUG [StoreOpener-500360dd67d4a36f45916b1c7a30f4aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:06:06,847 INFO  [StoreOpener-500360dd67d4a36f45916b1c7a30f4aa-1 {}] regionserver.HStore(327): Store=500360dd67d4a36f45916b1c7a30f4aa/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:06:06,849 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:06,849 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:06,853 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:06,856 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:06:06,857 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 500360dd67d4a36f45916b1c7a30f4aa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=861730, jitterRate=0.09574638307094574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:06:06,858 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 500360dd67d4a36f45916b1c7a30f4aa:

2024-12-09T17:06:06,859 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa., pid=11, masterSystemTime=1733763966830
2024-12-09T17:06:06,862 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,862 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:06,863 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=500360dd67d4a36f45916b1c7a30f4aa, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,40321,1733763964476
2024-12-09T17:06:06,868 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-09T17:06:06,869 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 500360dd67d4a36f45916b1c7a30f4aa, server=80c69eb3c456,40321,1733763964476 in 190 msec
2024-12-09T17:06:06,870 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-09T17:06:06,870 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=500360dd67d4a36f45916b1c7a30f4aa, ASSIGN in 352 msec
2024-12-09T17:06:06,871 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:06:06,871 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733763966871"}]},"ts":"1733763966871"}
2024-12-09T17:06:06,873 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-09T17:06:06,911 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:06:06,915 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 446 msec
2024-12-09T17:06:07,188 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:07,193 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:07,706 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:06:07,710 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:07,728 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:10,859 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-09T17:06:10,860 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-09T17:06:10,861 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-09T17:06:11,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-09T17:06:11,350 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-09T17:06:11,353 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-09T17:06:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44805 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:06:16,473 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath, procId: 9 completed
2024-12-09T17:06:16,478 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-09T17:06:16,478 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:16,491 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:06:16,493 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:16,519 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:16,533 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:16,537 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:16,538 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:16,538 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:16,538 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:06:16,539 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@186aeb49{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:16,539 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f2fc02f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:16,630 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59c9f3f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-38227-hadoop-hdfs-3_4_1-tests_jar-_-any-972654248312092388/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:16,631 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63c7640d{HTTP/1.1, (http/1.1)}{localhost:38227}
2024-12-09T17:06:16,631 INFO  [Time-limited test {}] server.Server(415): Started @131050ms
2024-12-09T17:06:16,632 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:06:16,662 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:16,665 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:16,666 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:16,666 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:16,666 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:06:16,667 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@588c20f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:16,667 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cc0980{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:16,760 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4fa1f5be{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-43805-hadoop-hdfs-3_4_1-tests_jar-_-any-12542651947621667319/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:16,761 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20121877{HTTP/1.1, (http/1.1)}{localhost:43805}
2024-12-09T17:06:16,761 INFO  [Time-limited test {}] server.Server(415): Started @131181ms
2024-12-09T17:06:16,762 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:06:16,788 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:16,790 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:16,791 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:16,791 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:16,791 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:06:16,792 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57b83639{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:16,792 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e3a97f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:16,883 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f3856ec{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-38923-hadoop-hdfs-3_4_1-tests_jar-_-any-6377989877193026623/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:16,884 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d00cc18{HTTP/1.1, (http/1.1)}{localhost:38923}
2024-12-09T17:06:16,884 INFO  [Time-limited test {}] server.Server(415): Started @131303ms
2024-12-09T17:06:16,885 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:06:17,621 WARN  [Thread-671 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data5/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:17,621 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data6/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:17,638 WARN  [Thread-612 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:06:17,640 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ad9f183f954beff with lease ID 0xaab4f63442d1482b: Processing first storage report for DS-c4958bf9-a430-4a87-939e-5b73fee5ce52 from datanode DatanodeRegistration(127.0.0.1:36821, datanodeUuid=ce2ec5c9-1076-4937-86af-ec1e867678f1, infoPort=35175, infoSecurePort=0, ipcPort=45637, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:17,640 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ad9f183f954beff with lease ID 0xaab4f63442d1482b: from storage DS-c4958bf9-a430-4a87-939e-5b73fee5ce52 node DatanodeRegistration(127.0.0.1:36821, datanodeUuid=ce2ec5c9-1076-4937-86af-ec1e867678f1, infoPort=35175, infoSecurePort=0, ipcPort=45637, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:17,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ad9f183f954beff with lease ID 0xaab4f63442d1482b: Processing first storage report for DS-5211381d-71d4-4232-924f-665e43102ff7 from datanode DatanodeRegistration(127.0.0.1:36821, datanodeUuid=ce2ec5c9-1076-4937-86af-ec1e867678f1, infoPort=35175, infoSecurePort=0, ipcPort=45637, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:17,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ad9f183f954beff with lease ID 0xaab4f63442d1482b: from storage DS-5211381d-71d4-4232-924f-665e43102ff7 node DatanodeRegistration(127.0.0.1:36821, datanodeUuid=ce2ec5c9-1076-4937-86af-ec1e867678f1, infoPort=35175, infoSecurePort=0, ipcPort=45637, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:17,823 WARN  [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data7/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:17,823 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data8/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:17,842 WARN  [Thread-634 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:06:17,845 WARN  [Thread-693 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:17,845 WARN  [Thread-694 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10/current/BP-787740456-172.17.0.2-1733763962753/current, will proceed with Du for space computation calculation, 
2024-12-09T17:06:17,845 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x14308749f8d39c6c with lease ID 0xaab4f63442d1482c: Processing first storage report for DS-e05230c9-e665-44ed-a500-0e515cdd1f36 from datanode DatanodeRegistration(127.0.0.1:41981, datanodeUuid=5d529ec0-9452-4fb8-98eb-cb2ea5ee2c39, infoPort=46187, infoSecurePort=0, ipcPort=37237, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:17,845 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x14308749f8d39c6c with lease ID 0xaab4f63442d1482c: from storage DS-e05230c9-e665-44ed-a500-0e515cdd1f36 node DatanodeRegistration(127.0.0.1:41981, datanodeUuid=5d529ec0-9452-4fb8-98eb-cb2ea5ee2c39, infoPort=46187, infoSecurePort=0, ipcPort=37237, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:17,845 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x14308749f8d39c6c with lease ID 0xaab4f63442d1482c: Processing first storage report for DS-c82f56da-ab5f-40be-98cd-f332e6de956c from datanode DatanodeRegistration(127.0.0.1:41981, datanodeUuid=5d529ec0-9452-4fb8-98eb-cb2ea5ee2c39, infoPort=46187, infoSecurePort=0, ipcPort=37237, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:17,845 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x14308749f8d39c6c with lease ID 0xaab4f63442d1482c: from storage DS-c82f56da-ab5f-40be-98cd-f332e6de956c node DatanodeRegistration(127.0.0.1:41981, datanodeUuid=5d529ec0-9452-4fb8-98eb-cb2ea5ee2c39, infoPort=46187, infoSecurePort=0, ipcPort=37237, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:17,860 WARN  [Thread-656 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:06:17,862 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3eee3fbef64956b4 with lease ID 0xaab4f63442d1482d: Processing first storage report for DS-7b96eda8-8da0-40a9-948e-04580cd2320b from datanode DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:17,862 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3eee3fbef64956b4 with lease ID 0xaab4f63442d1482d: from storage DS-7b96eda8-8da0-40a9-948e-04580cd2320b node DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:17,862 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3eee3fbef64956b4 with lease ID 0xaab4f63442d1482d: Processing first storage report for DS-84c487cd-0327-4fbb-89fc-63288d1a5b23 from datanode DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753)
2024-12-09T17:06:17,862 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3eee3fbef64956b4 with lease ID 0xaab4f63442d1482d: from storage DS-84c487cd-0327-4fbb-89fc-63288d1a5b23 node DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:17,923 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38f56d13{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:17,924 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d72d7be{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:17,924 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:17,924 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62cd5620{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:17,925 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63d13cc5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:17,919 WARN  [ResponseProcessor for block BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,919 WARN  [ResponseProcessor for block BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,919 WARN  [ResponseProcessor for block BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,928 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta block BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:17,928 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,43075,1733763966349/80c69eb3c456%2C43075%2C1733763966349.1733763966588 block BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:17,928 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 block BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK], DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:17,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:59806 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34177:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59806 dst: /127.0.0.1:34177
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_476917695_22 at /127.0.0.1:59878 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:34177:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59878 dst: /127.0.0.1:34177
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:59818 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34177:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59818 dst: /127.0.0.1:34177
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_476917695_22 at /127.0.0.1:57918 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:43139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57918 dst: /127.0.0.1:43139
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48667 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:57870 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57870 dst: /127.0.0.1:43139
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48946 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:57822 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57822 dst: /127.0.0.1:43139
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48988 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:57856 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43139:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57856 dst: /127.0.0.1:43139
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48933 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,919 WARN  [ResponseProcessor for block BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,928 WARN  [PacketResponder: BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43139] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,929 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 block BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:17,929 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:59780 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34177:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59780 dst: /127.0.0.1:34177
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:17,930 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:17,930 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:17,930 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:17,930 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-787740456-172.17.0.2-1733763962753 (Datanode Uuid a6304213-1985-48e0-b839-4b54bb0146cd) service to localhost/127.0.0.1:34441
2024-12-09T17:06:17,931 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data3/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:17,931 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data4/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:17,931 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:17,933 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 block BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,934 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 block BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,933 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,43075,1733763966349/80c69eb3c456%2C43075%2C1733763966349.1733763966588 block BP-787740456-172.17.0.2-1733763962753:blk_1073741839_1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,934 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta block BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,935 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c96e968{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:17,935 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21edda6b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:17,936 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:17,936 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22b28729{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:17,936 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b1c985{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:17,937 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:17,937 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:17,937 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:17,937 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-787740456-172.17.0.2-1733763962753 (Datanode Uuid 71d8b779-e8b0-4886-99d2-88560251626f) service to localhost/127.0.0.1:34441
2024-12-09T17:06:17,938 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data1/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:17,938 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data2/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:17,938 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:17,941 WARN  [RS:0;80c69eb3c456:40321.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=4, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,942 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C40321%2C1733763964476:(num 1733763965007) roll requested
2024-12-09T17:06:17,942 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.1733763977942
2024-12-09T17:06:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40321 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40321 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:37222 deadline: 1733763987941, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
2024-12-09T17:06:17,952 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
2024-12-09T17:06:17,952 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 with entries=4, filesize=959 B; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763977942
2024-12-09T17:06:17,954 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46187:46187),(127.0.0.1/127.0.0.1:35175:35175)]
2024-12-09T17:06:17,954 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:17,954 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,954 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:17,955 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-09T17:06:17,955 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-09T17:06:17,955 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007
2024-12-09T17:06:17,958 WARN  [IPC Server handler 0 on default port 34441 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 has not been closed. Lease recovery is in progress. RecoveryId = 1021 for block blk_1073741833_1009
2024-12-09T17:06:17,960 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 after 5ms
2024-12-09T17:06:21,962 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 after 4007ms
2024-12-09T17:06:30,024 INFO  [Time-limited test {}] wal.TestLogRolling(243): log.getCurrentFileName(): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763977942
2024-12-09T17:06:30,026 WARN  [ResponseProcessor for block BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:30,027 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763977942 block BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1020 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:30,028 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:47262 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:41981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47262 dst: /127.0.0.1:41981
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:30,029 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:38194 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:36821:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38194 dst: /127.0.0.1:36821
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:30,031 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4fa1f5be{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:30,032 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20121877{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:30,032 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:30,032 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cc0980{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:30,032 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@588c20f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:30,034 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:30,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:30,034 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-787740456-172.17.0.2-1733763962753 (Datanode Uuid 5d529ec0-9452-4fb8-98eb-cb2ea5ee2c39) service to localhost/127.0.0.1:34441
2024-12-09T17:06:30,034 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:30,035 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data7/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:30,035 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data8/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:30,035 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:30,037 WARN  [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]
2024-12-09T17:06:30,037 WARN  [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]
2024-12-09T17:06:30,037 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C40321%2C1733763964476:(num 1733763977942) roll requested
2024-12-09T17:06:30,038 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.1733763990038
2024-12-09T17:06:30,044 WARN  [Thread-713 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:30,043 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41702 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741841_1023 to mirror 127.0.0.1:43139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:30,044 WARN  [Thread-713 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:30,044 WARN  [Thread-713 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741841_1023
2024-12-09T17:06:30,044 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41702 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:30,044 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41702 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41702 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:30,046 WARN  [Thread-713 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]
2024-12-09T17:06:30,049 WARN  [Thread-713 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:30,049 WARN  [Thread-713 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]) is bad.
2024-12-09T17:06:30,049 WARN  [Thread-713 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741842_1024
2024-12-09T17:06:30,050 WARN  [Thread-713 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]
2024-12-09T17:06:30,053 WARN  [Thread-713 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41981
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:30,053 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:45696 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741843_1025] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data6]'}, localName='127.0.0.1:36821', datanodeUuid='ce2ec5c9-1076-4937-86af-ec1e867678f1', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741843_1025 to mirror 127.0.0.1:41981
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:30,054 WARN  [Thread-713 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:30,054 WARN  [Thread-713 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741843_1025
2024-12-09T17:06:30,054 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:45696 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741843_1025] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:30,054 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:45696 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:36821:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45696 dst: /127.0.0.1:36821
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:30,054 WARN  [Thread-713 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:30,060 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763977942 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763990038
2024-12-09T17:06:30,060 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35175:35175),(127.0.0.1/127.0.0.1:36959:36959)]
2024-12-09T17:06:30,060 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:30,060 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763977942 is not closed yet, will try archiving it next time
2024-12-09T17:06:30,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36821 is added to blk_1073741840_1022 (size=2431)
2024-12-09T17:06:30,464 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:30,661 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@af14e68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36821, datanodeUuid=ce2ec5c9-1076-4937-86af-ec1e867678f1, infoPort=35175, infoSecurePort=0, ipcPort=45637, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741840_1022 to 127.0.0.1:43139 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,043 WARN  [ResponseProcessor for block BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1026 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1026
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,044 WARN  [DataStreamer for file /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763990038 block BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:34,045 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:45702 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:36821:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45702 dst: /127.0.0.1:36821
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,046 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41708 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41708 dst: /127.0.0.1:41015
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,208 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59c9f3f9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:34,209 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63c7640d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:34,209 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:34,209 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f2fc02f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:34,210 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@186aeb49{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:34,213 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:34,213 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:34,213 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-787740456-172.17.0.2-1733763962753 (Datanode Uuid ce2ec5c9-1076-4937-86af-ec1e867678f1) service to localhost/127.0.0.1:34441
2024-12-09T17:06:34,213 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:34,214 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data5/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:34,215 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data6/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:34,215 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:34,217 WARN  [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]
2024-12-09T17:06:34,217 WARN  [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]
2024-12-09T17:06:34,217 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C40321%2C1733763964476:(num 1733763990038) roll requested
2024-12-09T17:06:34,218 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.1733763994217
2024-12-09T17:06:34,221 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,221 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK], DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]) is bad.
2024-12-09T17:06:34,221 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741845_1028
2024-12-09T17:06:34,222 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]
2024-12-09T17:06:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40321 {}] regionserver.HRegion(8581): Flush requested on 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:34,222 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 500360dd67d4a36f45916b1c7a30f4aa 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:06:34,224 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41981
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,224 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41714 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741846_1029 to mirror 127.0.0.1:41981
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,225 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:34,225 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741846_1029
2024-12-09T17:06:34,225 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41714 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:34,225 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41714 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41714 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,225 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:34,227 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,227 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:34,227 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741847_1030
2024-12-09T17:06:34,228 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]
2024-12-09T17:06:34,231 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36821
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,231 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41728 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741848_1031 to mirror 127.0.0.1:36821
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,231 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:34,231 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741848_1031
2024-12-09T17:06:34,231 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41728 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:34,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41728 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41728 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,232 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:34,233 WARN  [IPC Server handler 2 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-09T17:06:34,233 WARN  [IPC Server handler 2 on default port 34441 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-09T17:06:34,233 WARN  [IPC Server handler 2 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-09T17:06:34,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/24776f6bb3a845958a5561cd977e5b63 is 1080, key is row0002/info:/1733763990036/Put/seqid=0
2024-12-09T17:06:34,241 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763990038 with entries=13, filesize=14.10 KB; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994217
2024-12-09T17:06:34,241 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36959:36959)]
2024-12-09T17:06:34,242 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,242 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763990038 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,242 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34177
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,242 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41752 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741850_1033 to mirror 127.0.0.1:34177
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,242 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]) is bad.
2024-12-09T17:06:34,242 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741850_1033
2024-12-09T17:06:34,242 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41752 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-09T17:06:34,243 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41752 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41752 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,243 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]
2024-12-09T17:06:34,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741844_1027 (size=14443)
2024-12-09T17:06:34,244 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,244 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:34,244 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741851_1034
2024-12-09T17:06:34,245 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:34,246 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,246 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:34,246 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741852_1035
2024-12-09T17:06:34,246 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:34,248 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,248 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41760 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741853_1036 to mirror 127.0.0.1:43139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,248 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:34,248 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741853_1036
2024-12-09T17:06:34,248 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41760 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-09T17:06:34,249 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41760 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41760 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,249 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]
2024-12-09T17:06:34,250 WARN  [IPC Server handler 1 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-09T17:06:34,250 WARN  [IPC Server handler 1 on default port 34441 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-09T17:06:34,250 WARN  [IPC Server handler 1 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-09T17:06:34,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741854_1037 (size=10347)
2024-12-09T17:06:34,304 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:06:34,439 WARN  [sync.2 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]
2024-12-09T17:06:34,439 WARN  [sync.2 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]
2024-12-09T17:06:34,440 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C40321%2C1733763964476:(num 1733763994217) roll requested
2024-12-09T17:06:34,440 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.1733763994440
2024-12-09T17:06:34,448 WARN  [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43139
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,448 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41780 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741855_1038 to mirror 127.0.0.1:43139
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,449 WARN  [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:34,449 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41780 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:34,449 WARN  [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741855_1038
2024-12-09T17:06:34,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41780 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41780 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,450 WARN  [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]
2024-12-09T17:06:34,451 WARN  [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,452 WARN  [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:34,452 WARN  [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741856_1039
2024-12-09T17:06:34,452 WARN  [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:34,454 WARN  [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,454 WARN  [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]) is bad.
2024-12-09T17:06:34,454 WARN  [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741857_1040
2024-12-09T17:06:34,455 WARN  [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]
2024-12-09T17:06:34,458 WARN  [Thread-740 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41981
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,458 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41792 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741858_1041 to mirror 127.0.0.1:41981
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,458 WARN  [Thread-740 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:34,458 WARN  [Thread-740 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741858_1041
2024-12-09T17:06:34,458 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41792 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:34,458 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:41792 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41792 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,459 WARN  [Thread-740 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:34,460 WARN  [IPC Server handler 3 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-09T17:06:34,460 WARN  [IPC Server handler 3 on default port 34441 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-09T17:06:34,460 WARN  [IPC Server handler 3 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-09T17:06:34,464 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994217 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994440
2024-12-09T17:06:34,464 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36959:36959)]
2024-12-09T17:06:34,464 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,464 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763990038 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,464 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994217 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741849_1032 (size=1261)
2024-12-09T17:06:34,644 WARN  [sync.4 {}] wal.FSHLog(760): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-09T17:06:34,646 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,646 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994217 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,656 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/24776f6bb3a845958a5561cd977e5b63
2024-12-09T17:06:34,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/24776f6bb3a845958a5561cd977e5b63 as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/24776f6bb3a845958a5561cd977e5b63
2024-12-09T17:06:34,675 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/24776f6bb3a845958a5561cd977e5b63, entries=5, sequenceid=12, filesize=10.1 K
2024-12-09T17:06:34,676 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 500360dd67d4a36f45916b1c7a30f4aa in 454ms, sequenceid=12, compaction requested=false
2024-12-09T17:06:34,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 500360dd67d4a36f45916b1c7a30f4aa:

2024-12-09T17:06:34,788 WARN  [master/80c69eb3c456:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=96, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,789 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C44805%2C1733763964328:(num 1733763964596) roll requested
2024-12-09T17:06:34,790 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C44805%2C1733763964328.1733763994789
2024-12-09T17:06:34,790 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,791 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,796 WARN  [Thread-747 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,797 WARN  [Thread-747 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]) is bad.
2024-12-09T17:06:34,797 WARN  [Thread-747 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741860_1043
2024-12-09T17:06:34,798 WARN  [Thread-747 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]
2024-12-09T17:06:34,801 WARN  [Thread-747 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,801 WARN  [Thread-747 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]) is bad.
2024-12-09T17:06:34,801 WARN  [Thread-747 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741861_1044
2024-12-09T17:06:34,802 WARN  [Thread-747 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43139,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK]
2024-12-09T17:06:34,804 WARN  [Thread-747 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,804 WARN  [Thread-747 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:34,804 WARN  [Thread-747 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741862_1045
2024-12-09T17:06:34,805 WARN  [Thread-747 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:34,807 WARN  [Thread-747 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41981
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,807 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:41816 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741863_1046 to mirror 127.0.0.1:41981
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,807 WARN  [Thread-747 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:34,807 WARN  [Thread-747 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741863_1046
2024-12-09T17:06:34,807 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:41816 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:34,807 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:41816 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41816 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:34,808 WARN  [Thread-747 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:34,809 WARN  [IPC Server handler 3 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-09T17:06:34,809 WARN  [IPC Server handler 3 on default port 34441 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-09T17:06:34,809 WARN  [IPC Server handler 3 on default port 34441 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-09T17:06:34,812 WARN  [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
2024-12-09T17:06:34,812 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 with entries=93, filesize=46.04 KB; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763994789
2024-12-09T17:06:34,812 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36959:36959)]
2024-12-09T17:06:34,812 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,812 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,813 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:34,813 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596
2024-12-09T17:06:34,813 WARN  [IPC Server handler 1 on default port 34441 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 has not been closed. Lease recovery is in progress. RecoveryId = 1048 for block blk_1073741830_1006
2024-12-09T17:06:34,813 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 after 0ms
2024-12-09T17:06:34,836 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:06:34,837 INFO  [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54018, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:06:34,857 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:06:34,862 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:06:34,863 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:06:34,863 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:06:34,863 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:06:34,864 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e6b0aa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:06:34,864 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@720fd379{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:06:34,866 DEBUG [Close-WAL-Writer-2 {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763965007 is not closed yet, will try archiving it next time
2024-12-09T17:06:34,867 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763977942 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs/80c69eb3c456%2C40321%2C1733763964476.1733763977942
2024-12-09T17:06:34,952 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39e2672a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/java.io.tmpdir/jetty-localhost-34399-hadoop-hdfs-3_4_1-tests_jar-_-any-8854681320866520073/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:34,953 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29c97027{HTTP/1.1, (http/1.1)}{localhost:34399}
2024-12-09T17:06:34,953 INFO  [Time-limited test {}] server.Server(415): Started @149373ms
2024-12-09T17:06:34,954 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:06:35,288 WARN  [Thread-766 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:06:35,294 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa62e66c9d2d21b0f with lease ID 0xaab4f63442d1482e: from storage DS-bd814197-03fc-4480-b40d-da61c2c800df node DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:35,295 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa62e66c9d2d21b0f with lease ID 0xaab4f63442d1482e: from storage DS-868b3bc6-dd21-46a9-92b9-fdebd76e0ee7 node DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:06:36,873 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@157718b5[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741844_1027 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:36,873 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@11488ec3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741854_1037 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:37,403 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:06:37,408 INFO  [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54030, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:06:37,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741849_1032 (size=1261)
2024-12-09T17:06:38,817 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 after 4004ms
2024-12-09T17:06:40,294 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b7e8c51[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741838_1014 to 127.0.0.1:41981 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:40,295 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17beefe9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741836_1012 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:41,293 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17beefe9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741832_1008 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:41,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:06:43,294 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b7e8c51[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741829_1005 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:43,294 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17beefe9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741827_1003 to 127.0.0.1:41981 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:44,292 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17beefe9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741825_1001 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:45,311 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@26d26c42 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-787740456-172.17.0.2-1733763962753:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:34177,null,null])
java.net.ConnectException: Call From 80c69eb3c456/172.17.0.2 to localhost:37741 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-09T17:06:45,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741833_1021 (size=959)
2024-12-09T17:06:46,221 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-09T17:06:46,221 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-09T17:06:46,293 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17beefe9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741826_1002 to 127.0.0.1:41981 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:46,293 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b7e8c51[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741837_1013 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:47,294 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b7e8c51[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741835_1011 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:47,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:06:49,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741833_1021 (size=959)
2024-12-09T17:06:51,842 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 500360dd67d4a36f45916b1c7a30f4aa, had cached 0 bytes from a total of 10347
2024-12-09T17:06:53,648 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.1733764013647
2024-12-09T17:06:53,655 WARN  [Thread-796 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,655 WARN  [Thread-796 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741865_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:53,655 WARN  [Thread-796 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741865_1049
2024-12-09T17:06:53,656 WARN  [Thread-796 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:53,660 WARN  [Thread-796 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1050
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36821
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,660 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:55074 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741866_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data4]'}, localName='127.0.0.1:42905', datanodeUuid='a6304213-1985-48e0-b839-4b54bb0146cd', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741866_1050 to mirror 127.0.0.1:36821
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,660 WARN  [Thread-796 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741866_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42905,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:53,660 WARN  [Thread-796 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741866_1050
2024-12-09T17:06:53,660 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:55074 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741866_1050] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:53,660 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-572886139_22 at /127.0.0.1:55074 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741866_1050] {}] datanode.DataXceiver(331): 127.0.0.1:42905:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55074 dst: /127.0.0.1:42905
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,661 WARN  [Thread-796 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:53,668 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994440 with entries=2, filesize=1.57 KB; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733764013647
2024-12-09T17:06:53,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36959:36959),(127.0.0.1/127.0.0.1:41975:41975)]
2024-12-09T17:06:53,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994440 is not closed yet, will try archiving it next time
2024-12-09T17:06:53,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741859_1042 (size=1618)
2024-12-09T17:06:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40321 {}] regionserver.HRegion(8581): Flush requested on 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:53,671 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 500360dd67d4a36f45916b1c7a30f4aa 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-09T17:06:53,672 INFO  [sync.3 {}] wal.FSHLog(777): LowReplication-Roller was enabled.
2024-12-09T17:06:53,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/495d1a71213744c09c14b2508991a690 is 1080, key is row0007/info:/1733763994223/Put/seqid=0
2024-12-09T17:06:53,679 WARN  [Thread-803 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,680 WARN  [Thread-803 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741868_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:53,680 WARN  [Thread-803 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741868_1052
2024-12-09T17:06:53,680 WARN  [Thread-803 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:53,682 WARN  [Thread-803 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,682 WARN  [Thread-803 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741869_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:53,682 WARN  [Thread-803 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741869_1053
2024-12-09T17:06:53,682 WARN  [Thread-803 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:53,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741870_1054 (size=13583)
2024-12-09T17:06:53,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741870_1054 (size=13583)
2024-12-09T17:06:53,691 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=25 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/495d1a71213744c09c14b2508991a690
2024-12-09T17:06:53,691 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-09T17:06:53,691 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-09T17:06:53,691 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ad056f1 to 127.0.0.1:60386
2024-12-09T17:06:53,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:53,692 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-09T17:06:53,692 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1892698569, stopped=false
2024-12-09T17:06:53,692 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=80c69eb3c456,44805,1733763964328
2024-12-09T17:06:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/495d1a71213744c09c14b2508991a690 as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/495d1a71213744c09c14b2508991a690
2024-12-09T17:06:53,706 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/495d1a71213744c09c14b2508991a690, entries=8, sequenceid=25, filesize=13.3 K
2024-12-09T17:06:53,707 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~10.50 KB/10757, heapSize ~11.48 KB/11760, currentSize=9.46 KB/9684 for 500360dd67d4a36f45916b1c7a30f4aa in 36ms, sequenceid=25, compaction requested=false
2024-12-09T17:06:53,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 500360dd67d4a36f45916b1c7a30f4aa:

2024-12-09T17:06:53,707 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=23.4 K, sizeToCheck=16.0 K
2024-12-09T17:06:53,707 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:06:53,707 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/495d1a71213744c09c14b2508991a690 because midkey is the same as first or last row
2024-12-09T17:06:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:06:53,748 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-09T17:06:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:53,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:53,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:53,749 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,40321,1733763964476' *****
2024-12-09T17:06:53,749 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:06:53,749 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,43075,1733763966349' *****
2024-12-09T17:06:53,749 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:06:53,749 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:53,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:53,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:06:53,750 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:06:53,750 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:06:53,750 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:06:53,750 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:06:53,750 INFO  [RS:1;80c69eb3c456:43075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:06:53,751 INFO  [RS:1;80c69eb3c456:43075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:06:53,751 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,43075,1733763966349
2024-12-09T17:06:53,751 DEBUG [RS:1;80c69eb3c456:43075 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:53,751 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,43075,1733763966349; all regions closed.
2024-12-09T17:06:53,751 INFO  [RS:0;80c69eb3c456:40321 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:06:53,751 INFO  [RS:0;80c69eb3c456:40321 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:06:53,751 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(3579): Received CLOSE for 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:53,752 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,43075,1733763966349
2024-12-09T17:06:53,752 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(3579): Received CLOSE for 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:53,752 WARN  [WAL-Shutdown-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,752 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,40321,1733763964476
2024-12-09T17:06:53,752 DEBUG [RS:0;80c69eb3c456:40321 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:53,752 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 500360dd67d4a36f45916b1c7a30f4aa, disabling compactions & flushes
2024-12-09T17:06:53,753 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:53,753 ERROR [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1664): Shutdown / close of WAL failed: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
2024-12-09T17:06:53,753 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:53,753 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:06:53,753 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:06:53,753 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa. after waiting 0 ms
2024-12-09T17:06:53,753 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:06:53,753 DEBUG [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1665): Shutdown / close exception details:
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,753 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:53,753 DEBUG [RS:1;80c69eb3c456:43075 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:53,753 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:06:53,753 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:06:53,753 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 500360dd67d4a36f45916b1c7a30f4aa 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-09T17:06:53,754 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-09T17:06:53,754 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1603): Online Regions={500360dd67d4a36f45916b1c7a30f4aa=TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa., 1588230740=hbase:meta,,1.1588230740, 218a4c3d52fad4eee63ea39e5e84c0ca=hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.}
2024-12-09T17:06:53,754 INFO  [RS:1;80c69eb3c456:43075 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-09T17:06:53,754 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:06:53,754 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 218a4c3d52fad4eee63ea39e5e84c0ca, 500360dd67d4a36f45916b1c7a30f4aa
2024-12-09T17:06:53,754 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:06:53,754 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:06:53,754 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:06:53,754 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:06:53,754 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:06:53,754 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:06:53,754 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:06:53,754 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:06:53,754 INFO  [RS:1;80c69eb3c456:43075 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43075
2024-12-09T17:06:53,754 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.87 KB heapSize=5.40 KB
2024-12-09T17:06:53,755 WARN  [RS_OPEN_META-regionserver/80c69eb3c456:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,755 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C40321%2C1733763964476.meta:.meta(num 1733763965476) roll requested
2024-12-09T17:06:53,755 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:06:53,756 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C40321%2C1733763964476.meta.1733764013755.meta
2024-12-09T17:06:53,756 ERROR [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server 80c69eb3c456,40321,1733763964476: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,756 ERROR [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint]
2024-12-09T17:06:53,759 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory
2024-12-09T17:06:53,759 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/791049aceb0c4f37a31f3a4b15646933 is 1080, key is row0014/info:/1733764013672/Put/seqid=0
2024-12-09T17:06:53,759 WARN  [Thread-811 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1055
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36821
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:33956 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741871_1055] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741871_1055 to mirror 127.0.0.1:36821
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,759 WARN  [Thread-811 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741871_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:53,759 WARN  [Thread-811 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741871_1055
2024-12-09T17:06:53,760 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:33956 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741871_1055] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-09T17:06:53,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:33956 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741871_1055] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33956 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,760 WARN  [Thread-811 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:53,760 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC
2024-12-09T17:06:53,760 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication
2024-12-09T17:06:53,761 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server
2024-12-09T17:06:53,761 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: {
  "beans": [
    {
      "name": "java.lang:type=Memory",
      "modelerType": "sun.management.MemoryImpl",
      "ObjectPendingFinalizationCount": 0,
      "HeapMemoryUsage": {
        "committed": 1048576000,
        "init": 1048576000,
        "max": 2306867200,
        "used": 429291008
      },
      "NonHeapMemoryUsage": {
        "committed": 161939456,
        "init": 7667712,
        "max": -1,
        "used": 160192944
      },
      "Verbose": false,
      "ObjectName": "java.lang:type=Memory"
    }
  ],
  "beans": [],
  "beans": [],
  "beans": []
}
2024-12-09T17:06:53,761 WARN  [Thread-811 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,762 WARN  [Thread-811 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741873_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK], DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:53,762 WARN  [Thread-811 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741873_1057
2024-12-09T17:06:53,762 WARN  [Thread-812 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1056
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36821
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,762 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:55122 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741872_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data4]'}, localName='127.0.0.1:42905', datanodeUuid='a6304213-1985-48e0-b839-4b54bb0146cd', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741872_1056 to mirror 127.0.0.1:36821
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,762 WARN  [Thread-812 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741872_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42905,DS-bd814197-03fc-4480-b40d-da61c2c800df,DISK], DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]) is bad.
2024-12-09T17:06:53,762 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:55122 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741872_1056] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-09T17:06:53,762 WARN  [Thread-812 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741872_1056
2024-12-09T17:06:53,762 WARN  [Thread-811 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:53,762 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:55122 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741872_1056] {}] datanode.DataXceiver(331): 127.0.0.1:42905:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55122 dst: /127.0.0.1:42905
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,763 WARN  [Thread-812 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36821,DS-c4958bf9-a430-4a87-939e-5b73fee5ce52,DISK]
2024-12-09T17:06:53,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,43075,1733763966349
2024-12-09T17:06:53,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:06:53,764 WARN  [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44805 {}] master.MasterRpcServices(626): 80c69eb3c456,40321,1733763964476 reported a fatal error:
***** ABORTING region server 80c69eb3c456,40321,1733763964476: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
Cause:
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967)
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168)
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627)
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408)
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707)

2024-12-09T17:06:53,765 WARN  [Thread-812 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41981
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,765 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:33974 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741875_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10]'}, localName='127.0.0.1:41015', datanodeUuid='85b8d1e2-49bf-47df-ad1e-82e912cadd98', xmitsInProgress=0}:Exception transferring block BP-787740456-172.17.0.2-1733763962753:blk_1073741875_1059 to mirror 127.0.0.1:41981
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,765 WARN  [Thread-812 {}] hdfs.DataStreamer(1731): Error Recovery for BP-787740456-172.17.0.2-1733763962753:blk_1073741875_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK], DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]) is bad.
2024-12-09T17:06:53,765 WARN  [Thread-812 {}] hdfs.DataStreamer(1850): Abandoning BP-787740456-172.17.0.2-1733763962753:blk_1073741875_1059
2024-12-09T17:06:53,765 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:33974 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741875_1059] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-09T17:06:53,765 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1716849894_22 at /127.0.0.1:33974 [Receiving block BP-787740456-172.17.0.2-1733763962753:blk_1073741875_1059] {}] datanode.DataXceiver(331): 127.0.0.1:41015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33974 dst: /127.0.0.1:41015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:53,766 WARN  [Thread-812 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41981,DS-e05230c9-e665-44ed-a500-0e515cdd1f36,DISK]
2024-12-09T17:06:53,767 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
2024-12-09T17:06:53,767 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta with entries=11, filesize=3.63 KB; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733764013755.meta
2024-12-09T17:06:53,767 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36959:36959),(127.0.0.1/127.0.0.1:41975:41975)]
2024-12-09T17:06:53,767 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta is not closed yet, will try archiving it next time
2024-12-09T17:06:53,767 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,768 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34177,DS-be35cfe3-5010-4a88-96aa-c63a027da5a2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:06:53,768 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
2024-12-09T17:06:53,768 WARN  [IPC Server handler 2 on default port 34441 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta has not been closed. Lease recovery is in progress. RecoveryId = 1061 for block blk_1073741834_1010
2024-12-09T17:06:53,768 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta after 0ms
2024-12-09T17:06:53,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741876_1060 (size=14663)
2024-12-09T17:06:53,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741876_1060 (size=14663)
2024-12-09T17:06:53,772 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/791049aceb0c4f37a31f3a4b15646933
2024-12-09T17:06:53,772 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,43075,1733763966349]
2024-12-09T17:06:53,772 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,43075,1733763966349; numProcessing=1
2024-12-09T17:06:53,779 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/.tmp/info/791049aceb0c4f37a31f3a4b15646933 as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/791049aceb0c4f37a31f3a4b15646933
2024-12-09T17:06:53,781 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,43075,1733763966349 already deleted, retry=false
2024-12-09T17:06:53,781 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,43075,1733763966349 expired; onlineServers=1
2024-12-09T17:06:53,785 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/info/791049aceb0c4f37a31f3a4b15646933, entries=9, sequenceid=37, filesize=14.3 K
2024-12-09T17:06:53,786 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 500360dd67d4a36f45916b1c7a30f4aa in 33ms, sequenceid=37, compaction requested=true
2024-12-09T17:06:53,790 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/data/default/TestLogRolling-testLogRollOnDatanodeDeath/500360dd67d4a36f45916b1c7a30f4aa/recovered.edits/40.seqid, newMaxSeqId=40, maxSeqId=1
2024-12-09T17:06:53,791 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 500360dd67d4a36f45916b1c7a30f4aa:

2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733763966465.500360dd67d4a36f45916b1c7a30f4aa.
2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 218a4c3d52fad4eee63ea39e5e84c0ca, disabling compactions & flushes
2024-12-09T17:06:53,791 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca. after waiting 0 ms
2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 218a4c3d52fad4eee63ea39e5e84c0ca:

2024-12-09T17:06:53,791 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,869 INFO  [regionserver/80c69eb3c456:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-09T17:06:53,869 INFO  [regionserver/80c69eb3c456:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-09T17:06:53,873 INFO  [RS:1;80c69eb3c456:43075 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,43075,1733763966349; zookeeper connection closed.
2024-12-09T17:06:53,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:53,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43075-0x1000bc1ba140003, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:53,874 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49800016 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49800016
2024-12-09T17:06:53,954 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:06:53,954 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(3579): Received CLOSE for 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:53,955 DEBUG [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 218a4c3d52fad4eee63ea39e5e84c0ca
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 218a4c3d52fad4eee63ea39e5e84c0ca, disabling compactions & flushes
2024-12-09T17:06:53,955 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,955 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca. after waiting 0 ms
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 218a4c3d52fad4eee63ea39e5e84c0ca:

2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733763965569.218a4c3d52fad4eee63ea39e5e84c0ca.
2024-12-09T17:06:53,955 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740
2024-12-09T17:06:54,072 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763990038 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs/80c69eb3c456%2C40321%2C1733763964476.1733763990038
2024-12-09T17:06:54,077 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994217 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs/80c69eb3c456%2C40321%2C1733763964476.1733763994217
2024-12-09T17:06:54,080 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.1733763994440 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs/80c69eb3c456%2C40321%2C1733763964476.1733763994440
2024-12-09T17:06:54,155 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing
2024-12-09T17:06:54,155 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,40321,1733763964476; all regions closed.
2024-12-09T17:06:54,156 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476
2024-12-09T17:06:54,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741874_1058 (size=93)
2024-12-09T17:06:54,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741874_1058 (size=93)
2024-12-09T17:06:54,437 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:06:54,868 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:06:54,874 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@11488ec3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41015, datanodeUuid=85b8d1e2-49bf-47df-ad1e-82e912cadd98, infoPort=36959, infoSecurePort=0, ipcPort=43349, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741859_1042 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:55,324 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@26d26c42 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:34177,null,null])
java.net.ConnectException: Call From 80c69eb3c456/172.17.0.2 to localhost:37741 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-09T17:06:57,771 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta after 4003ms
2024-12-09T17:06:58,296 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17beefe9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42905, datanodeUuid=a6304213-1985-48e0-b839-4b54bb0146cd, infoPort=41975, infoSecurePort=0, ipcPort=37769, storageInfo=lv=-57;cid=testClusterID;nsid=869659752;c=1733763962753):Failed to transfer BP-787740456-172.17.0.2-1733763962753:blk_1073741830_1048 to 127.0.0.1:36821 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:06:58,793 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:58,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:58,814 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:58,815 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:58,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:58,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,160 ERROR [WAL-Shutdown-0 {}] wal.FSHLog(508): We have waited 5 seconds but the close of writer(s) doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-09T17:06:59,161 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476
2024-12-09T17:06:59,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741867_1051 (size=13514)
2024-12-09T17:06:59,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741867_1051 (size=13514)
2024-12-09T17:06:59,169 DEBUG [RS:0;80c69eb3c456:40321 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:59,169 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:06:59,170 INFO  [RS:0;80c69eb3c456:40321 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-09T17:06:59,170 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:06:59,170 INFO  [RS:0;80c69eb3c456:40321 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40321
2024-12-09T17:06:59,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,40321,1733763964476
2024-12-09T17:06:59,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:06:59,231 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,40321,1733763964476]
2024-12-09T17:06:59,231 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,40321,1733763964476; numProcessing=2
2024-12-09T17:06:59,239 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,40321,1733763964476 already deleted, retry=false
2024-12-09T17:06:59,239 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,40321,1733763964476 expired; onlineServers=0
2024-12-09T17:06:59,239 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,44805,1733763964328' *****
2024-12-09T17:06:59,239 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-09T17:06:59,240 DEBUG [M:0;80c69eb3c456:44805 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2682413b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:06:59,240 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,44805,1733763964328
2024-12-09T17:06:59,240 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,44805,1733763964328; all regions closed.
2024-12-09T17:06:59,240 DEBUG [M:0;80c69eb3c456:44805 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:06:59,240 DEBUG [M:0;80c69eb3c456:44805 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-09T17:06:59,241 DEBUG [M:0;80c69eb3c456:44805 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-09T17:06:59,241 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-09T17:06:59,241 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733763964789 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733763964789,5,FailOnTimeoutGroup]
2024-12-09T17:06:59,241 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733763964788 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733763964788,5,FailOnTimeoutGroup]
2024-12-09T17:06:59,241 INFO  [M:0;80c69eb3c456:44805 {}] hbase.ChoreService(370): Chore service for: master/80c69eb3c456:0 had [] on shutdown
2024-12-09T17:06:59,242 DEBUG [M:0;80c69eb3c456:44805 {}] master.HMaster(1733): Stopping service threads
2024-12-09T17:06:59,242 INFO  [M:0;80c69eb3c456:44805 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-09T17:06:59,242 INFO  [M:0;80c69eb3c456:44805 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-09T17:06:59,243 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-09T17:06:59,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-09T17:06:59,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:06:59,252 DEBUG [M:0;80c69eb3c456:44805 {}] zookeeper.ZKUtil(347): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-09T17:06:59,252 WARN  [M:0;80c69eb3c456:44805 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-09T17:06:59,252 INFO  [M:0;80c69eb3c456:44805 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-09T17:06:59,252 INFO  [M:0;80c69eb3c456:44805 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-09T17:06:59,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:06:59,253 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:06:59,253 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:59,253 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:59,253 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:06:59,253 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:59,253 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.29 KB
2024-12-09T17:06:59,255 WARN  [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]
2024-12-09T17:06:59,255 WARN  [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41015,DS-7b96eda8-8da0-40a9-948e-04580cd2320b,DISK]]
2024-12-09T17:06:59,255 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C44805%2C1733763964328:(num 1733763994789) roll requested
2024-12-09T17:06:59,256 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C44805%2C1733763964328.1733764019256
2024-12-09T17:06:59,267 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763994789 with entries=1, filesize=349 B; new WAL /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733764019256
2024-12-09T17:06:59,267 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36959:36959),(127.0.0.1/127.0.0.1:41975:41975)]
2024-12-09T17:06:59,267 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763994789 is not closed yet, will try archiving it next time
2024-12-09T17:06:59,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741864_1047 (size=357)
2024-12-09T17:06:59,276 DEBUG [M:0;80c69eb3c456:44805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/829e8722c7df4ce2adf118cc24f705ad is 82, key is hbase:meta,,1/info:regioninfo/1733763965502/Put/seqid=0
2024-12-09T17:06:59,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741878_1063 (size=5672)
2024-12-09T17:06:59,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741878_1063 (size=5672)
2024-12-09T17:06:59,282 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/829e8722c7df4ce2adf118cc24f705ad
2024-12-09T17:06:59,302 DEBUG [M:0;80c69eb3c456:44805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b1dfd48ea9ba4035a66aac38fcc71eaf is 774, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733763966913/Put/seqid=0
2024-12-09T17:06:59,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741879_1064 (size=7465)
2024-12-09T17:06:59,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741879_1064 (size=7465)
2024-12-09T17:06:59,308 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.41 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b1dfd48ea9ba4035a66aac38fcc71eaf
2024-12-09T17:06:59,326 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:06:59,327 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,328 DEBUG [M:0;80c69eb3c456:44805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4b02101249b8493ebc6c17c5887d36de is 69, key is 80c69eb3c456,40321,1733763964476/rs:state/1733763964841/Put/seqid=0
2024-12-09T17:06:59,331 INFO  [RS:0;80c69eb3c456:40321 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,40321,1733763964476; zookeeper connection closed.
2024-12-09T17:06:59,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:59,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40321-0x1000bc1ba140001, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:59,331 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e8045ed {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e8045ed
2024-12-09T17:06:59,331 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-09T17:06:59,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741880_1065 (size=5224)
2024-12-09T17:06:59,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741880_1065 (size=5224)
2024-12-09T17:06:59,334 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4b02101249b8493ebc6c17c5887d36de
2024-12-09T17:06:59,341 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,341 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,345 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:06:59,352 DEBUG [M:0;80c69eb3c456:44805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef8f1a540a104982aa1a2a969bb1948c is 52, key is load_balancer_on/state:d/1733763966330/Put/seqid=0
2024-12-09T17:06:59,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741881_1066 (size=5056)
2024-12-09T17:06:59,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741881_1066 (size=5056)
2024-12-09T17:06:59,357 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef8f1a540a104982aa1a2a969bb1948c
2024-12-09T17:06:59,362 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/829e8722c7df4ce2adf118cc24f705ad as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/829e8722c7df4ce2adf118cc24f705ad
2024-12-09T17:06:59,368 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/829e8722c7df4ce2adf118cc24f705ad, entries=8, sequenceid=97, filesize=5.5 K
2024-12-09T17:06:59,369 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b1dfd48ea9ba4035a66aac38fcc71eaf as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b1dfd48ea9ba4035a66aac38fcc71eaf
2024-12-09T17:06:59,374 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b1dfd48ea9ba4035a66aac38fcc71eaf, entries=11, sequenceid=97, filesize=7.3 K
2024-12-09T17:06:59,375 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4b02101249b8493ebc6c17c5887d36de as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4b02101249b8493ebc6c17c5887d36de
2024-12-09T17:06:59,381 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4b02101249b8493ebc6c17c5887d36de, entries=2, sequenceid=97, filesize=5.1 K
2024-12-09T17:06:59,383 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef8f1a540a104982aa1a2a969bb1948c as hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ef8f1a540a104982aa1a2a969bb1948c
2024-12-09T17:06:59,390 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ef8f1a540a104982aa1a2a969bb1948c, entries=1, sequenceid=97, filesize=4.9 K
2024-12-09T17:06:59,391 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41039, heapSize ~49.23 KB/50408, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 138ms, sequenceid=97, compaction requested=false
2024-12-09T17:06:59,393 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:06:59,393 DEBUG [M:0;80c69eb3c456:44805 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:06:59,393 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328
2024-12-09T17:06:59,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42905 is added to blk_1073741877_1062 (size=493)
2024-12-09T17:06:59,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41015 is added to blk_1073741877_1062 (size=493)
2024-12-09T17:06:59,672 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763964596 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/oldWALs/80c69eb3c456%2C44805%2C1733763964328.1733763964596
2024-12-09T17:06:59,682 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/oldWALs/80c69eb3c456%2C44805%2C1733763964328.1733763964596 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs/80c69eb3c456%2C44805%2C1733763964328.1733763964596$masterlocalwal$
2024-12-09T17:06:59,682 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/WALs/80c69eb3c456,44805,1733763964328/80c69eb3c456%2C44805%2C1733763964328.1733763994789 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/oldWALs/80c69eb3c456%2C44805%2C1733763964328.1733763994789
2024-12-09T17:06:59,687 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/MasterData/oldWALs/80c69eb3c456%2C44805%2C1733763964328.1733763994789 to hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/oldWALs/80c69eb3c456%2C44805%2C1733763964328.1733763994789$masterlocalwal$
2024-12-09T17:06:59,688 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:06:59,688 INFO  [M:0;80c69eb3c456:44805 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-09T17:06:59,688 INFO  [M:0;80c69eb3c456:44805 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44805
2024-12-09T17:06:59,731 DEBUG [M:0;80c69eb3c456:44805 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/80c69eb3c456,44805,1733763964328 already deleted, retry=false
2024-12-09T17:06:59,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:06:59,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:59,840 INFO  [M:0;80c69eb3c456:44805 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,44805,1733763964328; zookeeper connection closed.
2024-12-09T17:06:59,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44805-0x1000bc1ba140000, quorum=127.0.0.1:60386, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:06:59,847 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39e2672a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:59,848 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29c97027{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:59,848 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:59,848 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@720fd379{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:59,849 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e6b0aa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:59,850 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:59,850 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:59,850 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-787740456-172.17.0.2-1733763962753 (Datanode Uuid a6304213-1985-48e0-b839-4b54bb0146cd) service to localhost/127.0.0.1:34441
2024-12-09T17:06:59,850 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:59,850 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@73f8514c {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:34177,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:37741 , LocalHost:localPort 80c69eb3c456/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-09T17:06:59,851 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@73f8514c {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-787740456-172.17.0.2-1733763962753:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:42905,null,null], DatanodeInfoWithStorage[127.0.0.1:34177,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-787740456-172.17.0.2-1733763962753
2024-12-09T17:06:59,851 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data3/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:59,851 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data4/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:59,852 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:59,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f3856ec{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:06:59,854 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d00cc18{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:59,854 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:59,854 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e3a97f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:59,854 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57b83639{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:59,855 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:06:59,855 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:06:59,855 WARN  [BP-787740456-172.17.0.2-1733763962753 heartbeating to localhost/127.0.0.1:34441 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-787740456-172.17.0.2-1733763962753 (Datanode Uuid 85b8d1e2-49bf-47df-ad1e-82e912cadd98) service to localhost/127.0.0.1:34441
2024-12-09T17:06:59,855 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:06:59,856 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data9/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:59,856 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/cluster_123fea45-1994-5f28-302f-6b34cb324ec9/dfs/data/data10/current/BP-787740456-172.17.0.2-1733763962753 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:06:59,856 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:06:59,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@381717fc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:06:59,861 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43db7010{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:06:59,861 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:06:59,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56fd08b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:06:59,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a43de3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir/,STOPPED}
2024-12-09T17:06:59,868 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-09T17:06:59,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-09T17:06:59,904 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=88 (was 66)
Potentially hanging thread: RS-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:34441 from jenkins.hfs.1
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34441
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:34441
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34441
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:34441 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34441
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$$Lambda$797/0x00007ff41cb83870.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Abort regionserver monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-14-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:34441
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34441
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:34441 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=424 (was 403) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=50 (was 77), ProcessCount=11 (was 11), AvailableMemoryMB=5129 (was 5348)
2024-12-09T17:06:59,909 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=88, OpenFileDescriptor=424, MaxFileDescriptor=1048576, SystemLoadAverage=50, ProcessCount=11, AvailableMemoryMB=5129
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.log.dir so I do NOT create it in target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5059c8fe-f08c-4c24-e64a-3498a95c5dce/hadoop.tmp.dir so I do NOT create it in target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845, deleteOnExit=true
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/test.cache.data in system properties and HBase conf
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.tmp.dir in system properties and HBase conf
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir in system properties and HBase conf
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-09T17:06:59,910 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-09T17:06:59,911 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:06:59,911 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-09T17:06:59,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/nfs.dump.dir in system properties and HBase conf
2024-12-09T17:06:59,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir in system properties and HBase conf
2024-12-09T17:06:59,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:06:59,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-09T17:06:59,912 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-09T17:06:59,923 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:07:00,208 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:00,212 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:00,213 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:00,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:00,214 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:07:00,214 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:00,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@140f7a56{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:00,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68473501{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:00,302 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ecc9f79{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-37965-hadoop-hdfs-3_4_1-tests_jar-_-any-15414363573230772022/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:07:00,303 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@708ba1ca{HTTP/1.1, (http/1.1)}{localhost:37965}
2024-12-09T17:07:00,303 INFO  [Time-limited test {}] server.Server(415): Started @174722ms
2024-12-09T17:07:00,312 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:07:00,481 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:00,484 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:00,485 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:00,485 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:00,485 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:07:00,485 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70c26223{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:00,486 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31dced60{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:00,573 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4aa46864{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-39375-hadoop-hdfs-3_4_1-tests_jar-_-any-18446625337963718864/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:00,574 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@534139ce{HTTP/1.1, (http/1.1)}{localhost:39375}
2024-12-09T17:07:00,574 INFO  [Time-limited test {}] server.Server(415): Started @174993ms
2024-12-09T17:07:00,575 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:00,597 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:00,600 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:00,601 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:00,601 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:00,601 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:07:00,602 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31be664c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:00,602 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2070a2a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:00,691 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4fd89e84{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-45179-hadoop-hdfs-3_4_1-tests_jar-_-any-10182016807821741793/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:00,691 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@783c49af{HTTP/1.1, (http/1.1)}{localhost:45179}
2024-12-09T17:07:00,691 INFO  [Time-limited test {}] server.Server(415): Started @175111ms
2024-12-09T17:07:00,692 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:00,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:01,212 WARN  [Thread-953 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data1/current/BP-495047503-172.17.0.2-1733764019933/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:01,212 WARN  [Thread-954 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data2/current/BP-495047503-172.17.0.2-1733764019933/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:01,232 WARN  [Thread-917 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:01,234 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x54680db8e977e9fe with lease ID 0x30948081af0e4461: Processing first storage report for DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3 from datanode DatanodeRegistration(127.0.0.1:41151, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=36339, infoSecurePort=0, ipcPort=35081, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933)
2024-12-09T17:07:01,234 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x54680db8e977e9fe with lease ID 0x30948081af0e4461: from storage DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3 node DatanodeRegistration(127.0.0.1:41151, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=36339, infoSecurePort=0, ipcPort=35081, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:01,234 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x54680db8e977e9fe with lease ID 0x30948081af0e4461: Processing first storage report for DS-d7482d78-f43f-4144-885b-09520400b4b3 from datanode DatanodeRegistration(127.0.0.1:41151, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=36339, infoSecurePort=0, ipcPort=35081, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933)
2024-12-09T17:07:01,234 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x54680db8e977e9fe with lease ID 0x30948081af0e4461: from storage DS-d7482d78-f43f-4144-885b-09520400b4b3 node DatanodeRegistration(127.0.0.1:41151, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=36339, infoSecurePort=0, ipcPort=35081, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:01,295 WARN  [Thread-964 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data3/current/BP-495047503-172.17.0.2-1733764019933/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:01,295 WARN  [Thread-965 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data4/current/BP-495047503-172.17.0.2-1733764019933/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:01,315 WARN  [Thread-940 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:01,316 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb47742961c797316 with lease ID 0x30948081af0e4462: Processing first storage report for DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda from datanode DatanodeRegistration(127.0.0.1:37273, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=37003, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933)
2024-12-09T17:07:01,317 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb47742961c797316 with lease ID 0x30948081af0e4462: from storage DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda node DatanodeRegistration(127.0.0.1:37273, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=37003, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:01,317 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb47742961c797316 with lease ID 0x30948081af0e4462: Processing first storage report for DS-b2d05143-1608-4ad8-9f48-7e7924cb35d3 from datanode DatanodeRegistration(127.0.0.1:37273, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=37003, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933)
2024-12-09T17:07:01,317 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb47742961c797316 with lease ID 0x30948081af0e4462: from storage DS-b2d05143-1608-4ad8-9f48-7e7924cb35d3 node DatanodeRegistration(127.0.0.1:37273, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=37003, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:01,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-09T17:07:01,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-09T17:07:01,419 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775
2024-12-09T17:07:01,424 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/zookeeper_0, clientPort=51082, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-09T17:07:01,426 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51082
2024-12-09T17:07:01,426 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,427 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:07:01,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:07:01,437 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693 with version=8
2024-12-09T17:07:01,437 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase-staging
2024-12-09T17:07:01,439 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:07:01,439 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:01,439 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:01,439 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:07:01,439 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:01,440 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:07:01,440 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:07:01,440 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:07:01,440 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41375
2024-12-09T17:07:01,441 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,442 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,444 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:41375 connecting to ZooKeeper ensemble=127.0.0.1:51082
2024-12-09T17:07:01,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413750x0, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:07:01,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41375-0x1000bc2992f0000 connected
2024-12-09T17:07:01,581 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:07:01,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:07:01,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:07:01,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41375
2024-12-09T17:07:01,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41375
2024-12-09T17:07:01,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41375
2024-12-09T17:07:01,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41375
2024-12-09T17:07:01,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41375
2024-12-09T17:07:01,585 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693, hbase.cluster.distributed=false
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:07:01,608 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:07:01,609 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36387
2024-12-09T17:07:01,609 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:07:01,610 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:07:01,610 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,611 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,613 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36387 connecting to ZooKeeper ensemble=127.0.0.1:51082
2024-12-09T17:07:01,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363870x0, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:07:01,622 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:07:01,622 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36387-0x1000bc2992f0001 connected
2024-12-09T17:07:01,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:07:01,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:07:01,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36387
2024-12-09T17:07:01,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36387
2024-12-09T17:07:01,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36387
2024-12-09T17:07:01,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36387
2024-12-09T17:07:01,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36387
2024-12-09T17:07:01,626 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:01,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:01,635 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,638 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80c69eb3c456:41375
2024-12-09T17:07:01,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:07:01,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:07:01,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,643 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:07:01,644 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80c69eb3c456,41375,1733764021439 from backup master directory
2024-12-09T17:07:01,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:01,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:01,651 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:07:01,651 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:07:01,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:07:01,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:07:01,664 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/hbase.id with ID: d5ef0bfd-b8dd-4756-b574-e3ef623f21a7
2024-12-09T17:07:01,678 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:01,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:07:01,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:07:01,699 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:07:01,700 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-09T17:07:01,701 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:07:01,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:07:01,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:07:01,710 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store
2024-12-09T17:07:01,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:07:01,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:07:01,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:01,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:07:01,718 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:01,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:01,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:07:01,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:01,718 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:01,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:07:01,719 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/.initializing
2024-12-09T17:07:01,719 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,722 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C41375%2C1733764021439, suffix=, logDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439, archiveDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/oldWALs, maxLogs=10
2024-12-09T17:07:01,722 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41375%2C1733764021439.1733764021722
2024-12-09T17:07:01,728 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722
2024-12-09T17:07:01,728 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37003:37003),(127.0.0.1/127.0.0.1:36339:36339)]
2024-12-09T17:07:01,728 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:01,729 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:01,729 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:01,729 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:01,730 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:01,732 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-09T17:07:01,732 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,732 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:01,732 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:01,734 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-09T17:07:01,734 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,734 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:01,734 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:01,736 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-09T17:07:01,736 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,736 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:01,736 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:01,738 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-09T17:07:01,738 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,738 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:01,739 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:01,739 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:01,741 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-09T17:07:01,743 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:01,745 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:01,746 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=740952, jitterRate=-0.05783109366893768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-09T17:07:01,746 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:07:01,747 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-09T17:07:01,751 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4220e702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:07:01,752 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-09T17:07:01,752 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-09T17:07:01,752 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-09T17:07:01,752 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-09T17:07:01,753 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-09T17:07:01,753 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-09T17:07:01,753 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-09T17:07:01,755 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-09T17:07:01,756 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-09T17:07:01,763 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-09T17:07:01,764 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-09T17:07:01,764 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-09T17:07:01,772 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-09T17:07:01,772 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-09T17:07:01,774 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-09T17:07:01,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:01,780 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-09T17:07:01,782 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-09T17:07:01,789 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-09T17:07:01,793 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-09T17:07:01,801 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-09T17:07:01,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:07:01,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:07:01,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,811 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=80c69eb3c456,41375,1733764021439, sessionid=0x1000bc2992f0000, setting cluster-up flag (Was=false)
2024-12-09T17:07:01,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,855 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-09T17:07:01,856 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:01,897 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-09T17:07:01,899 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,41375,1733764021439
2024-12-09T17:07:01,903 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-09T17:07:01,903 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-09T17:07:01,904 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80c69eb3c456,41375,1733764021439
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80c69eb3c456:0, corePoolSize=10, maxPoolSize=10
2024-12-09T17:07:01,904 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,905 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:07:01,905 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733764051906
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-09T17:07:01,906 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-09T17:07:01,907 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:07:01,907 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-09T17:07:01,907 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,908 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-09T17:07:01,908 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-09T17:07:01,908 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-09T17:07:01,908 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,908 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-09T17:07:01,909 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-09T17:07:01,908 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:07:01,909 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764021909,5,FailOnTimeoutGroup]
2024-12-09T17:07:01,909 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764021909,5,FailOnTimeoutGroup]
2024-12-09T17:07:01,909 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,909 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-09T17:07:01,909 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,909 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:07:01,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:07:01,918 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-09T17:07:01,919 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693
2024-12-09T17:07:01,925 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:07:01,925 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:07:01,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:01,927 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:07:01,928 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:07:01,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,929 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:01,929 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:07:01,930 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:07:01,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,931 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:01,931 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:07:01,932 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:07:01,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:01,933 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:01,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740
2024-12-09T17:07:01,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740
2024-12-09T17:07:01,935 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:07:01,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:07:01,938 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80c69eb3c456:36387
2024-12-09T17:07:01,938 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:01,939 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872004, jitterRate=0.10881146788597107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:07:01,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:07:01,939 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1008): ClusterId : d5ef0bfd-b8dd-4756-b574-e3ef623f21a7
2024-12-09T17:07:01,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:07:01,939 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:07:01,939 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:07:01,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:07:01,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:07:01,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:07:01,939 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:07:01,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:07:01,940 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:07:01,940 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-09T17:07:01,940 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-09T17:07:01,941 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-09T17:07:01,942 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-09T17:07:01,947 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:07:01,948 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:07:01,956 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:07:01,956 DEBUG [RS:0;80c69eb3c456:36387 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718c9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:07:01,957 DEBUG [RS:0;80c69eb3c456:36387 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bcf232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:07:01,957 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:07:01,957 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:07:01,957 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:07:01,957 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,41375,1733764021439 with isa=80c69eb3c456/172.17.0.2:36387, startcode=1733764021607
2024-12-09T17:07:01,957 DEBUG [RS:0;80c69eb3c456:36387 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:07:01,959 INFO  [RS-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40027, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:07:01,959 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:01,959 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41375 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:01,961 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693
2024-12-09T17:07:01,961 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37483
2024-12-09T17:07:01,961 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:07:01,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:07:01,968 DEBUG [RS:0;80c69eb3c456:36387 {}] zookeeper.ZKUtil(111): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,36387,1733764021607
2024-12-09T17:07:01,968 WARN  [RS:0;80c69eb3c456:36387 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:07:01,968 INFO  [RS:0;80c69eb3c456:36387 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:07:01,969 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607
2024-12-09T17:07:01,969 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,36387,1733764021607]
2024-12-09T17:07:01,971 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:07:01,971 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:07:01,973 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:07:01,973 INFO  [RS:0;80c69eb3c456:36387 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:07:01,973 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,973 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:07:01,974 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:01,974 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:07:01,975 DEBUG [RS:0;80c69eb3c456:36387 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:07:01,976 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,976 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,976 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,976 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:01,976 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36387,1733764021607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:07:01,990 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:07:01,990 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36387,1733764021607-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:02,002 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.Replication(204): 80c69eb3c456,36387,1733764021607 started
2024-12-09T17:07:02,002 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,36387,1733764021607, RpcServer on 80c69eb3c456/172.17.0.2:36387, sessionid=0x1000bc2992f0001
2024-12-09T17:07:02,003 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:07:02,003 DEBUG [RS:0;80c69eb3c456:36387 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:02,003 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,36387,1733764021607'
2024-12-09T17:07:02,003 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:07:02,003 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,36387,1733764021607'
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:07:02,004 DEBUG [RS:0;80c69eb3c456:36387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:07:02,005 INFO  [RS:0;80c69eb3c456:36387 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:07:02,005 INFO  [RS:0;80c69eb3c456:36387 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:07:02,092 WARN  [80c69eb3c456:41375 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-09T17:07:02,108 INFO  [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C36387%2C1733764021607, suffix=, logDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607, archiveDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/oldWALs, maxLogs=32
2024-12-09T17:07:02,110 INFO  [RS:0;80c69eb3c456:36387 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:02,119 INFO  [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:02,119 DEBUG [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36339:36339),(127.0.0.1/127.0.0.1:37003:37003)]
2024-12-09T17:07:02,343 DEBUG [80c69eb3c456:41375 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-09T17:07:02,344 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:02,347 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,36387,1733764021607, state=OPENING
2024-12-09T17:07:02,389 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-09T17:07:02,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:02,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:02,399 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=80c69eb3c456,36387,1733764021607}]
2024-12-09T17:07:02,399 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:02,399 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:02,555 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:02,555 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-09T17:07:02,557 INFO  [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51664, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-09T17:07:02,562 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-09T17:07:02,562 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:07:02,565 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C36387%2C1733764021607.meta, suffix=.meta, logDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607, archiveDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/oldWALs, maxLogs=32
2024-12-09T17:07:02,567 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta
2024-12-09T17:07:02,576 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta
2024-12-09T17:07:02,576 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37003:37003),(127.0.0.1/127.0.0.1:36339:36339)]
2024-12-09T17:07:02,576 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:02,577 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-09T17:07:02,577 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-09T17:07:02,577 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-09T17:07:02,577 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-09T17:07:02,577 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:02,577 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-09T17:07:02,577 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-09T17:07:02,579 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:07:02,579 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:07:02,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:02,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:02,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:07:02,581 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:07:02,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:02,581 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:02,581 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:07:02,582 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:07:02,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:02,583 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:02,583 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740
2024-12-09T17:07:02,585 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740
2024-12-09T17:07:02,586 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:07:02,588 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:07:02,589 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=725949, jitterRate=-0.07690936326980591}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:07:02,589 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:07:02,589 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733764022555
2024-12-09T17:07:02,591 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-09T17:07:02,591 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-09T17:07:02,592 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:02,593 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,36387,1733764021607, state=OPEN
2024-12-09T17:07:02,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:07:02,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:07:02,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:02,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:02,621 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-09T17:07:02,621 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=80c69eb3c456,36387,1733764021607 in 219 msec
2024-12-09T17:07:02,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-09T17:07:02,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 681 msec
2024-12-09T17:07:02,625 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 723 msec
2024-12-09T17:07:02,625 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733764022625, completionTime=-1
2024-12-09T17:07:02,625 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-09T17:07:02,625 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-09T17:07:02,626 DEBUG [hconnection-0x1506932b-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:07:02,627 INFO  [RS-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51666, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:07:02,628 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-09T17:07:02,628 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733764082628
2024-12-09T17:07:02,628 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733764142628
2024-12-09T17:07:02,628 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-09T17:07:02,652 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41375,1733764021439-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:02,652 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41375,1733764021439-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:02,652 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41375,1733764021439-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:02,652 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80c69eb3c456:41375, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:02,652 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:02,653 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-09T17:07:02,653 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:07:02,655 DEBUG [master/80c69eb3c456:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-09T17:07:02,655 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-09T17:07:02,657 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:07:02,657 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:02,659 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:07:02,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:07:02,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:07:02,673 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4b4ce617a32ce6991d195b2772ff6e63, NAME => 'hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693
2024-12-09T17:07:02,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:07:02,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:07:02,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:02,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 4b4ce617a32ce6991d195b2772ff6e63, disabling compactions & flushes
2024-12-09T17:07:02,683 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:02,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:02,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63. after waiting 0 ms
2024-12-09T17:07:02,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:02,683 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:02,683 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4b4ce617a32ce6991d195b2772ff6e63:

2024-12-09T17:07:02,684 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:07:02,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733764022684"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764022684"}]},"ts":"1733764022684"}
2024-12-09T17:07:02,686 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:07:02,687 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:07:02,687 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764022687"}]},"ts":"1733764022687"}
2024-12-09T17:07:02,689 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-09T17:07:02,706 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=4b4ce617a32ce6991d195b2772ff6e63, ASSIGN}]
2024-12-09T17:07:02,707 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=4b4ce617a32ce6991d195b2772ff6e63, ASSIGN
2024-12-09T17:07:02,709 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=4b4ce617a32ce6991d195b2772ff6e63, ASSIGN; state=OFFLINE, location=80c69eb3c456,36387,1733764021607; forceNewPlan=false, retain=false
2024-12-09T17:07:02,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:02,860 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=4b4ce617a32ce6991d195b2772ff6e63, regionState=OPENING, regionLocation=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:02,865 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 4b4ce617a32ce6991d195b2772ff6e63, server=80c69eb3c456,36387,1733764021607}]
2024-12-09T17:07:03,022 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:03,031 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:03,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 4b4ce617a32ce6991d195b2772ff6e63, NAME => 'hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:03,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:03,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:03,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:03,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:03,034 INFO  [StoreOpener-4b4ce617a32ce6991d195b2772ff6e63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4b4ce617a32ce6991d195b2772ff6e63 
2024-12-09T17:07:03,036 INFO  [StoreOpener-4b4ce617a32ce6991d195b2772ff6e63-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b4ce617a32ce6991d195b2772ff6e63 columnFamilyName info
2024-12-09T17:07:03,036 DEBUG [StoreOpener-4b4ce617a32ce6991d195b2772ff6e63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:03,037 INFO  [StoreOpener-4b4ce617a32ce6991d195b2772ff6e63-1 {}] regionserver.HStore(327): Store=4b4ce617a32ce6991d195b2772ff6e63/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:03,038 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:03,038 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:03,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:03,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:03,045 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 4b4ce617a32ce6991d195b2772ff6e63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=774686, jitterRate=-0.014936268329620361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:07:03,045 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 4b4ce617a32ce6991d195b2772ff6e63:

2024-12-09T17:07:03,046 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63., pid=6, masterSystemTime=1733764023022
2024-12-09T17:07:03,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:03,048 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:03,049 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=4b4ce617a32ce6991d195b2772ff6e63, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:03,054 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-09T17:07:03,054 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 4b4ce617a32ce6991d195b2772ff6e63, server=80c69eb3c456,36387,1733764021607 in 187 msec
2024-12-09T17:07:03,058 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-09T17:07:03,058 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=4b4ce617a32ce6991d195b2772ff6e63, ASSIGN in 349 msec
2024-12-09T17:07:03,059 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:07:03,059 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764023059"}]},"ts":"1733764023059"}
2024-12-09T17:07:03,061 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-09T17:07:03,073 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:07:03,075 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 420 msec
2024-12-09T17:07:03,157 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-09T17:07:03,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:07:03,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:03,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:03,175 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-09T17:07:03,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:07:03,205 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec
2024-12-09T17:07:03,218 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-09T17:07:03,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:07:03,249 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 31 msec
2024-12-09T17:07:03,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-09T17:07:03,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-09T17:07:03,293 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.642sec
2024-12-09T17:07:03,293 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-09T17:07:03,293 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-09T17:07:03,293 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-09T17:07:03,293 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-09T17:07:03,293 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-09T17:07:03,294 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41375,1733764021439-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:07:03,294 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41375,1733764021439-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-09T17:07:03,297 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-09T17:07:03,297 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-09T17:07:03,297 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,41375,1733764021439-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:03,330 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26a12806 to 127.0.0.1:51082 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5aa9c50
2024-12-09T17:07:03,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38749a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:07:03,342 DEBUG [hconnection-0x69edc9c7-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:07:03,343 INFO  [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51678, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:07:03,345 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=80c69eb3c456,41375,1733764021439
2024-12-09T17:07:03,345 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:03,348 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-09T17:07:03,348 INFO  [Time-limited test {}] wal.TestLogRolling(297): Starting testLogRollOnPipelineRestart
2024-12-09T17:07:03,348 INFO  [Time-limited test {}] wal.TestLogRolling(300): Replication=2
2024-12-09T17:07:03,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-09T17:07:03,350 INFO  [RS-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50726, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-09T17:07:03,351 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-09T17:07:03,351 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-09T17:07:03,352 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:07:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-09T17:07:03,354 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:07:03,354 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:03,354 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 9
2024-12-09T17:07:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:07:03,355 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:07:03,361 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741837_1013 (size=395)
2024-12-09T17:07:03,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741837_1013 (size=395)
2024-12-09T17:07:03,364 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 17fe7a114ab44257fb9a3dc89de20bfd, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693
2024-12-09T17:07:03,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41151 is added to blk_1073741838_1014 (size=78)
2024-12-09T17:07:03,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37273 is added to blk_1073741838_1014 (size=78)
2024-12-09T17:07:03,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:03,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1681): Closing 17fe7a114ab44257fb9a3dc89de20bfd, disabling compactions & flushes
2024-12-09T17:07:03,371 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd. after waiting 0 ms
2024-12-09T17:07:03,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,371 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1635): Region close journal for 17fe7a114ab44257fb9a3dc89de20bfd:

2024-12-09T17:07:03,373 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:07:03,373 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733764023373"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764023373"}]},"ts":"1733764023373"}
2024-12-09T17:07:03,375 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:07:03,375 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:07:03,376 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764023376"}]},"ts":"1733764023376"}
2024-12-09T17:07:03,377 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-09T17:07:03,393 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=17fe7a114ab44257fb9a3dc89de20bfd, ASSIGN}]
2024-12-09T17:07:03,394 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=17fe7a114ab44257fb9a3dc89de20bfd, ASSIGN
2024-12-09T17:07:03,395 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=17fe7a114ab44257fb9a3dc89de20bfd, ASSIGN; state=OFFLINE, location=80c69eb3c456,36387,1733764021607; forceNewPlan=false, retain=false
2024-12-09T17:07:03,546 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=17fe7a114ab44257fb9a3dc89de20bfd, regionState=OPENING, regionLocation=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:03,550 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 17fe7a114ab44257fb9a3dc89de20bfd, server=80c69eb3c456,36387,1733764021607}]
2024-12-09T17:07:03,705 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:03,713 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,714 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 17fe7a114ab44257fb9a3dc89de20bfd, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:03,714 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:03,714 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:03,714 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:03,714 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:03,717 INFO  [StoreOpener-17fe7a114ab44257fb9a3dc89de20bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 17fe7a114ab44257fb9a3dc89de20bfd 
2024-12-09T17:07:03,719 INFO  [StoreOpener-17fe7a114ab44257fb9a3dc89de20bfd-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17fe7a114ab44257fb9a3dc89de20bfd columnFamilyName info
2024-12-09T17:07:03,719 DEBUG [StoreOpener-17fe7a114ab44257fb9a3dc89de20bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:03,720 INFO  [StoreOpener-17fe7a114ab44257fb9a3dc89de20bfd-1 {}] regionserver.HStore(327): Store=17fe7a114ab44257fb9a3dc89de20bfd/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:03,721 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:03,722 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:03,725 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:03,727 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:03,728 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 17fe7a114ab44257fb9a3dc89de20bfd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732334, jitterRate=-0.06879013776779175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:07:03,729 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 17fe7a114ab44257fb9a3dc89de20bfd:

2024-12-09T17:07:03,730 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd., pid=11, masterSystemTime=1733764023704
2024-12-09T17:07:03,732 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,732 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:03,733 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=17fe7a114ab44257fb9a3dc89de20bfd, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,36387,1733764021607
2024-12-09T17:07:03,738 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-09T17:07:03,738 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 17fe7a114ab44257fb9a3dc89de20bfd, server=80c69eb3c456,36387,1733764021607 in 185 msec
2024-12-09T17:07:03,741 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-09T17:07:03,741 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=17fe7a114ab44257fb9a3dc89de20bfd, ASSIGN in 345 msec
2024-12-09T17:07:03,742 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:07:03,742 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764023742"}]},"ts":"1733764023742"}
2024-12-09T17:07:03,744 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-09T17:07:03,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:03,791 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:07:03,796 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 439 msec
2024-12-09T17:07:04,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:05,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:06,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:06,858 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:07:06,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,880 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:06,882 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:07,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:07,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-09T17:07:08,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:09,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:10,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:11,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-09T17:07:11,349 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-09T17:07:11,351 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-09T17:07:11,351 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-09T17:07:11,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:12,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41375 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:07:13,356 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart, procId: 9 completed
2024-12-09T17:07:13,359 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-09T17:07:13,359 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:13,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:14,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:15,364 INFO  [Time-limited test {}] wal.TestLogRolling(337): log.getCurrentFileName()): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:15,365 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:15,365 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:15,365 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:15,366 WARN  [DataStreamer for file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722 block BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]) is bad.
2024-12-09T17:07:15,366 WARN  [DataStreamer for file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta block BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK], DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]) is bad.
2024-12-09T17:07:15,366 WARN  [DataStreamer for file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 block BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK], DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37273,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]) is bad.
2024-12-09T17:07:15,366 WARN  [PacketResponder: BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37273] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,366 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:58456 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58456 dst: /127.0.0.1:37273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,367 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:56276 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41151:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56276 dst: /127.0.0.1:41151
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,367 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_932747542_22 at /127.0.0.1:56228 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41151:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56228 dst: /127.0.0.1:41151
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,367 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:56274 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41151:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56274 dst: /127.0.0.1:41151
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,367 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:58440 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58440 dst: /127.0.0.1:37273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,368 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_932747542_22 at /127.0.0.1:58416 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58416 dst: /127.0.0.1:37273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,444 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4fd89e84{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:15,444 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@783c49af{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:15,444 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:15,445 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2070a2a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:15,445 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31be664c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:15,447 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:07:15,447 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:07:15,447 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495047503-172.17.0.2-1733764019933 (Datanode Uuid d1e7194a-829c-46db-a47a-b36801bd8363) service to localhost/127.0.0.1:37483
2024-12-09T17:07:15,447 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:07:15,448 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data3/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:15,448 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data4/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:15,449 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:07:15,458 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:15,461 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:15,462 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:15,462 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:15,462 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:07:15,462 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bc459ac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:15,463 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@367f1ef4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:15,551 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd5066d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-33323-hadoop-hdfs-3_4_1-tests_jar-_-any-4928201378178555519/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:15,552 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a5cbb85{HTTP/1.1, (http/1.1)}{localhost:33323}
2024-12-09T17:07:15,552 INFO  [Time-limited test {}] server.Server(415): Started @189971ms
2024-12-09T17:07:15,553 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:15,566 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:15,566 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:15,566 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1017 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1017
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:15,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:38718 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41151:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38718 dst: /127.0.0.1:41151
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_932747542_22 at /127.0.0.1:38734 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41151:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38734 dst: /127.0.0.1:41151
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:38720 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41151:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38720 dst: /127.0.0.1:41151
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:15,568 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4aa46864{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:15,569 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@534139ce{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:15,569 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:15,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31dced60{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:15,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70c26223{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:15,570 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:07:15,570 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:07:15,570 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495047503-172.17.0.2-1733764019933 (Datanode Uuid 299775b1-6b47-45a6-8426-21a4db1c3ca7) service to localhost/127.0.0.1:37483
2024-12-09T17:07:15,570 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:07:15,570 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data1/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:15,570 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data2/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:15,571 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:07:15,577 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:15,580 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:15,580 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:15,580 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:15,580 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:07:15,581 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a17e0ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:15,581 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d76faef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:15,670 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49e61742{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-37711-hadoop-hdfs-3_4_1-tests_jar-_-any-15795822779662797450/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:15,670 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13aa7d37{HTTP/1.1, (http/1.1)}{localhost:37711}
2024-12-09T17:07:15,671 INFO  [Time-limited test {}] server.Server(415): Started @190090ms
2024-12-09T17:07:15,672 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:15,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:15,907 WARN  [Thread-1099 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:15,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x975f0b56c2f070d1 with lease ID 0x30948081af0e4463: from storage DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda node DatanodeRegistration(127.0.0.1:43297, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=39451, infoSecurePort=0, ipcPort=34843, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-09T17:07:15,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x975f0b56c2f070d1 with lease ID 0x30948081af0e4463: from storage DS-b2d05143-1608-4ad8-9f48-7e7924cb35d3 node DatanodeRegistration(127.0.0.1:43297, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=39451, infoSecurePort=0, ipcPort=34843, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:16,040 WARN  [Thread-1119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:16,042 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4c69a8d0f65e6bd with lease ID 0x30948081af0e4464: from storage DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3 node DatanodeRegistration(127.0.0.1:41075, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=34435, infoSecurePort=0, ipcPort=45589, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:16,042 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4c69a8d0f65e6bd with lease ID 0x30948081af0e4464: from storage DS-d7482d78-f43f-4144-885b-09520400b4b3 node DatanodeRegistration(127.0.0.1:41075, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=34435, infoSecurePort=0, ipcPort=45589, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:16,689 INFO  [Time-limited test {}] wal.TestLogRolling(349): Data Nodes restarted
2024-12-09T17:07:16,692 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-09T17:07:16,693 WARN  [RS:0;80c69eb3c456:36387.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=5, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:16,693 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C36387%2C1733764021607:(num 1733764022109) roll requested
2024-12-09T17:07:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:16,693 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51678 deadline: 1733764046692, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
2024-12-09T17:07:16,699 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 newFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:16,699 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
2024-12-09T17:07:16,699 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 with entries=5, filesize=2.09 KB; new WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:16,700 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39451:39451),(127.0.0.1/127.0.0.1:34435:34435)]
2024-12-09T17:07:16,701 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 is not closed yet, will try archiving it next time
2024-12-09T17:07:16,701 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:16,701 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:16,701 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:16,702 WARN  [IPC Server handler 2 on default port 37483 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1015
2024-12-09T17:07:16,702 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 after 1ms
2024-12-09T17:07:16,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:17,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:18,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:18,912 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-09T17:07:19,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:20,704 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 after 4003ms
2024-12-09T17:07:20,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:21,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:22,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:23,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:24,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:25,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:26,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:27,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:28,746 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-09T17:07:28,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:29,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:30,751 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018
java.io.IOException: Bad response ERROR for BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018 from datanode DatanodeInfoWithStorage[127.0.0.1:41075,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:30,752 WARN  [DataStreamer for file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 block BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43297,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK], DatanodeInfoWithStorage[127.0.0.1:41075,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41075,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]) is bad.
2024-12-09T17:07:30,752 WARN  [PacketResponder: BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41075] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:30,753 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:45676 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:43297:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45676 dst: /127.0.0.1:43297
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:30,753 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:35846 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:41075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35846 dst: /127.0.0.1:41075
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:30,796 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49e61742{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:30,796 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13aa7d37{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:30,797 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:30,797 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d76faef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:30,797 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a17e0ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:30,799 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:07:30,799 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:07:30,799 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495047503-172.17.0.2-1733764019933 (Datanode Uuid 299775b1-6b47-45a6-8426-21a4db1c3ca7) service to localhost/127.0.0.1:37483
2024-12-09T17:07:30,799 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:07:30,800 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data1/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:30,800 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data2/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:30,800 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:07:30,810 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:30,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:30,813 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:30,813 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:30,813 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:30,813 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:07:30,814 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef7e454{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:30,815 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@bb3bbf0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:30,906 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@505145a5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-46543-hadoop-hdfs-3_4_1-tests_jar-_-any-5564778250359092107/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:30,906 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16267e26{HTTP/1.1, (http/1.1)}{localhost:46543}
2024-12-09T17:07:30,907 INFO  [Time-limited test {}] server.Server(415): Started @205326ms
2024-12-09T17:07:30,908 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:30,923 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:30,924 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1412260369_22 at /127.0.0.1:43604 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:43297:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43604 dst: /127.0.0.1:43297
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:30,925 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd5066d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:30,925 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a5cbb85{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:30,925 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:30,926 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@367f1ef4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:30,926 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bc459ac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:30,927 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:07:30,927 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:07:30,927 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:07:30,927 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495047503-172.17.0.2-1733764019933 (Datanode Uuid d1e7194a-829c-46db-a47a-b36801bd8363) service to localhost/127.0.0.1:37483
2024-12-09T17:07:30,927 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data3/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:30,927 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data4/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:30,928 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:07:30,935 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:30,939 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:30,939 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:30,940 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:30,940 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:07:30,940 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@148b9396{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:30,941 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2dc68e60{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:31,033 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15c82a5e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/java.io.tmpdir/jetty-localhost-46823-hadoop-hdfs-3_4_1-tests_jar-_-any-9708237373055841449/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:31,033 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6bfee350{HTTP/1.1, (http/1.1)}{localhost:46823}
2024-12-09T17:07:31,033 INFO  [Time-limited test {}] server.Server(415): Started @205453ms
2024-12-09T17:07:31,034 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:31,186 WARN  [Thread-1174 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:31,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60165f4b7476bd46 with lease ID 0x30948081af0e4465: from storage DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3 node DatanodeRegistration(127.0.0.1:39595, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=41985, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:31,189 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60165f4b7476bd46 with lease ID 0x30948081af0e4465: from storage DS-d7482d78-f43f-4144-885b-09520400b4b3 node DatanodeRegistration(127.0.0.1:39595, datanodeUuid=299775b1-6b47-45a6-8426-21a4db1c3ca7, infoPort=41985, infoSecurePort=0, ipcPort=40995, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:31,331 WARN  [Thread-1194 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:31,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x169091698f78aff8 with lease ID 0x30948081af0e4466: from storage DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda node DatanodeRegistration(127.0.0.1:34301, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=39617, infoSecurePort=0, ipcPort=42283, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:31,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x169091698f78aff8 with lease ID 0x30948081af0e4466: from storage DS-b2d05143-1608-4ad8-9f48-7e7924cb35d3 node DatanodeRegistration(127.0.0.1:34301, datanodeUuid=d1e7194a-829c-46db-a47a-b36801bd8363, infoPort=39617, infoSecurePort=0, ipcPort=42283, storageInfo=lv=-57;cid=testClusterID;nsid=1845808216;c=1733764019933), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:31,419 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:07:31,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:31,907 WARN  [master/80c69eb3c456:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=95, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:31,907 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C41375%2C1733764021439:(num 1733764021722) roll requested
2024-12-09T17:07:31,907 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:31,907 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:31,907 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C41375%2C1733764021439.1733764051907
2024-12-09T17:07:31,914 WARN  [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
2024-12-09T17:07:31,914 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722 with entries=92, filesize=45.99 KB; new WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764051907
2024-12-09T17:07:31,915 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39617:39617),(127.0.0.1/127.0.0.1:41985:41985)]
2024-12-09T17:07:31,915 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722 is not closed yet, will try archiving it next time
2024-12-09T17:07:31,915 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:31,915 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:31,915 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722
2024-12-09T17:07:31,916 WARN  [IPC Server handler 0 on default port 37483 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741830_1017
2024-12-09T17:07:31,916 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722 after 1ms
2024-12-09T17:07:31,957 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:07:31,958 INFO  [RS-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38432, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:07:32,050 INFO  [Time-limited test {}] wal.TestLogRolling(366): Data Nodes restarted
2024-12-09T17:07:32,051 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-09T17:07:32,052 WARN  [RS:0;80c69eb3c456:36387.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=8, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43297,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:32,053 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C36387%2C1733764021607:(num 1733764036693) roll requested
2024-12-09T17:07:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43297,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:32,053 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.1733764052053
2024-12-09T17:07:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51678 deadline: 1733764062052, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
2024-12-09T17:07:32,059 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 newFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053
2024-12-09T17:07:32,060 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
2024-12-09T17:07:32,060 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053
2024-12-09T17:07:32,060 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41985:41985),(127.0.0.1/127.0.0.1:39617:39617)]
2024-12-09T17:07:32,060 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 is not closed yet, will try archiving it next time
2024-12-09T17:07:32,060 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43297,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:32,060 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43297,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:32,060 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:32,061 WARN  [IPC Server handler 0 on default port 37483 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741839_1020
2024-12-09T17:07:32,061 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 after 1ms
2024-12-09T17:07:32,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:33,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:34,337 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 2) replica(s):
0) Failed to delete replica blk_1073741830_1017: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-09T17:07:34,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:35,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:35,918 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439/80c69eb3c456%2C41375%2C1733764021439.1733764021722 after 4003ms
2024-12-09T17:07:36,063 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 after 4003ms
2024-12-09T17:07:36,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:37,191 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741839_1020: GenerationStamp not matched, existing replica is blk_1073741839_1018
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-09T17:07:37,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:38,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:39,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:40,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:41,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:42,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:43,301 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-09T17:07:43,301 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-09T17:07:43,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:44,146 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:44,156 DEBUG [Time-limited test {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053 newFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:44,158 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:44,158 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39617:39617),(127.0.0.1/127.0.0.1:41985:41985)]
2024-12-09T17:07:44,158 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053 is not closed yet, will try archiving it next time
2024-12-09T17:07:44,158 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:44,159 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:44,159 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109 after 0ms
2024-12-09T17:07:44,159 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:44,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741841_1023 (size=1264)
2024-12-09T17:07:44,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741841_1023 (size=1264)
2024-12-09T17:07:44,169 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733764023045/Put/vlen=162/seqid=0]
2024-12-09T17:07:44,169 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [default/info:d/1733764023181/Put/vlen=9/seqid=0]
2024-12-09T17:07:44,169 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #5: [hbase/info:d/1733764023228/Put/vlen=7/seqid=0]
2024-12-09T17:07:44,170 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733764023729/Put/vlen=218/seqid=0]
2024-12-09T17:07:44,170 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [row1002/info:/1733764033362/Put/vlen=1045/seqid=0]
2024-12-09T17:07:44,170 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764022109
2024-12-09T17:07:44,170 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:44,170 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:44,171 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693 after 0ms
2024-12-09T17:07:44,171 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:44,174 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #6: [row1003/info:/1733764046742/Put/vlen=1045/seqid=0]
2024-12-09T17:07:44,174 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #7: [row1004/info:/1733764048748/Put/vlen=1045/seqid=0]
2024-12-09T17:07:44,174 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764036693
2024-12-09T17:07:44,174 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053
2024-12-09T17:07:44,174 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053
2024-12-09T17:07:44,175 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053 after 0ms
2024-12-09T17:07:44,175 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764052053
2024-12-09T17:07:44,178 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #9: [row1005/info:/1733764062144/Put/vlen=1045/seqid=0]
2024-12-09T17:07:44,178 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:44,178 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:44,178 WARN  [IPC Server handler 3 on default port 37483 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741842_1025
2024-12-09T17:07:44,179 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 after 1ms
2024-12-09T17:07:44,338 WARN  [ResponseProcessor for block BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:44,338 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_932747542_22 at /127.0.0.1:58094 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:34301:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58094 dst: /127.0.0.1:34301
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34301 remote=/127.0.0.1:58094]. Total timeout mills is 60000, 59818 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:44,338 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_932747542_22 at /127.0.0.1:36804 [Receiving block BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:39595:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36804 dst: /127.0.0.1:39595
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:07:44,339 WARN  [DataStreamer for file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 block BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34301,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK], DatanodeInfoWithStorage[127.0.0.1:39595,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34301,DS-74fe5d1b-3738-49ca-a8ca-d18a5b20fcda,DISK]) is bad.
2024-12-09T17:07:44,345 WARN  [DataStreamer for file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 block BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:44,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741842_1026 (size=85)
2024-12-09T17:07:44,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741842_1026 (size=85)
2024-12-09T17:07:44,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:45,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:46,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:47,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:48,180 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 after 4002ms
2024-12-09T17:07:48,181 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:48,189 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:48,190 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 17fe7a114ab44257fb9a3dc89de20bfd 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-09T17:07:48,190 WARN  [RS:0;80c69eb3c456:36387.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,191 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C36387%2C1733764021607:(num 1733764064146) roll requested
2024-12-09T17:07:48,191 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 17fe7a114ab44257fb9a3dc89de20bfd:

2024-12-09T17:07:48,191 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.1733764068191
2024-12-09T17:07:48,191 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,191 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.42 KB
2024-12-09T17:07:48,192 WARN  [RS_OPEN_META-regionserver/80c69eb3c456:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,192 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-09T17:07:48,192 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,192 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 4b4ce617a32ce6991d195b2772ff6e63 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-09T17:07:48,192 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 4b4ce617a32ce6991d195b2772ff6e63:

2024-12-09T17:07:48,192 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-09T17:07:48,196 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-09T17:07:48,196 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x26a12806 to 127.0.0.1:51082
2024-12-09T17:07:48,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:07:48,196 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-09T17:07:48,196 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2037570568, stopped=false
2024-12-09T17:07:48,196 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=80c69eb3c456,41375,1733764021439
2024-12-09T17:07:48,196 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 newFile=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764068191
2024-12-09T17:07:48,197 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
2024-12-09T17:07:48,197 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764068191
2024-12-09T17:07:48,197 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41985:41985),(127.0.0.1/127.0.0.1:39617:39617)]
2024-12-09T17:07:48,197 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 is not closed yet, will try archiving it next time
2024-12-09T17:07:48,197 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 80c69eb3c456%2C36387%2C1733764021607.meta:.meta(num 1733764022567) roll requested
2024-12-09T17:07:48,197 INFO  [regionserver/80c69eb3c456:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36387%2C1733764021607.meta.1733764068197.meta
2024-12-09T17:07:48,197 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,197 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-495047503-172.17.0.2-1733764019933:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,197 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:48,198 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 after 1ms
2024-12-09T17:07:48,199 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.1733764064146 to hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/oldWALs/80c69eb3c456%2C36387%2C1733764021607.1733764064146
2024-12-09T17:07:48,205 WARN  [regionserver/80c69eb3c456:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
2024-12-09T17:07:48,205 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta with entries=11, filesize=3.66 KB; new WAL /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764068197.meta
2024-12-09T17:07:48,205 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41985:41985),(127.0.0.1/127.0.0.1:39617:39617)]
2024-12-09T17:07:48,206 DEBUG [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta is not closed yet, will try archiving it next time
2024-12-09T17:07:48,206 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,206 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41151,DS-7018d0e7-f6d3-4f5f-8197-fbf9474fd4c3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-09T17:07:48,206 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta
2024-12-09T17:07:48,206 WARN  [IPC Server handler 3 on default port 37483 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta has not been closed. Lease recovery is in progress. RecoveryId = 1029 for block blk_1073741834_1016
2024-12-09T17:07:48,206 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta after 0ms
2024-12-09T17:07:48,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:07:48,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:07:48,262 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-09T17:07:48,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:48,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:48,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:07:48,263 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,36387,1733764021607' *****
2024-12-09T17:07:48,263 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:07:48,263 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:07:48,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:07:48,264 INFO  [RS:0;80c69eb3c456:36387 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:07:48,264 INFO  [RS:0;80c69eb3c456:36387 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:07:48,264 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(3579): Received CLOSE for 17fe7a114ab44257fb9a3dc89de20bfd
2024-12-09T17:07:48,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:07:48,264 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:07:48,265 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(3579): Received CLOSE for 4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:48,265 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,36387,1733764021607
2024-12-09T17:07:48,265 DEBUG [RS:0;80c69eb3c456:36387 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:07:48,265 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 17fe7a114ab44257fb9a3dc89de20bfd, disabling compactions & flushes
2024-12-09T17:07:48,265 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:48,265 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:07:48,265 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:48,265 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:07:48,265 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd. after waiting 0 ms
2024-12-09T17:07:48,266 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:48,266 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:07:48,266 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:07:48,266 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 17fe7a114ab44257fb9a3dc89de20bfd 1/1 column families, dataSize=4.20 KB heapSize=4.98 KB
2024-12-09T17:07:48,266 WARN  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-09T17:07:48,266 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-09T17:07:48,266 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1603): Online Regions={17fe7a114ab44257fb9a3dc89de20bfd=TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd., 1588230740=hbase:meta,,1.1588230740, 4b4ce617a32ce6991d195b2772ff6e63=hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.}
2024-12-09T17:07:48,266 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:07:48,267 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:07:48,267 DEBUG [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 17fe7a114ab44257fb9a3dc89de20bfd, 4b4ce617a32ce6991d195b2772ff6e63
2024-12-09T17:07:48,267 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:07:48,267 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:07:48,267 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:07:48,267 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.89 KB
2024-12-09T17:07:48,268 WARN  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-09T17:07:48,268 WARN  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-09T17:07:48,283 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/.tmp/info/27522d9f7c264282ae8b608273e79474 is 1080, key is row1002/info:/1733764033362/Put/seqid=0
2024-12-09T17:07:48,283 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/.tmp/info/3b1f8b49b7654eb283d36ec84ec1e12c is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd./info:regioninfo/1733764023733/Put/seqid=0
2024-12-09T17:07:48,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741845_1030 (size=9270)
2024-12-09T17:07:48,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741845_1030 (size=9270)
2024-12-09T17:07:48,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741846_1031 (size=8268)
2024-12-09T17:07:48,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741846_1031 (size=8268)
2024-12-09T17:07:48,288 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/.tmp/info/27522d9f7c264282ae8b608273e79474
2024-12-09T17:07:48,288 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/.tmp/info/3b1f8b49b7654eb283d36ec84ec1e12c
2024-12-09T17:07:48,294 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/.tmp/info/27522d9f7c264282ae8b608273e79474 as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/info/27522d9f7c264282ae8b608273e79474
2024-12-09T17:07:48,301 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/info/27522d9f7c264282ae8b608273e79474, entries=4, sequenceid=12, filesize=9.1 K
2024-12-09T17:07:48,302 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 17fe7a114ab44257fb9a3dc89de20bfd in 36ms, sequenceid=12, compaction requested=false
2024-12-09T17:07:48,307 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/default/TestLogRolling-testLogRollOnPipelineRestart/17fe7a114ab44257fb9a3dc89de20bfd/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1
2024-12-09T17:07:48,307 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:48,307 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 17fe7a114ab44257fb9a3dc89de20bfd:

2024-12-09T17:07:48,307 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733764023351.17fe7a114ab44257fb9a3dc89de20bfd.
2024-12-09T17:07:48,308 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4b4ce617a32ce6991d195b2772ff6e63, disabling compactions & flushes
2024-12-09T17:07:48,308 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:48,308 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:48,308 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63. after waiting 0 ms
2024-12-09T17:07:48,308 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:48,308 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4b4ce617a32ce6991d195b2772ff6e63 1/1 column families, dataSize=78 B heapSize=728 B
2024-12-09T17:07:48,308 WARN  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-09T17:07:48,309 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/.tmp/table/cf5f1567b68243139b1df0fb0aa78def is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733764023742/Put/seqid=0
2024-12-09T17:07:48,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741847_1032 (size=5482)
2024-12-09T17:07:48,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741847_1032 (size=5482)
2024-12-09T17:07:48,315 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=244 B at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/.tmp/table/cf5f1567b68243139b1df0fb0aa78def
2024-12-09T17:07:48,322 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/.tmp/info/3b1f8b49b7654eb283d36ec84ec1e12c as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/info/3b1f8b49b7654eb283d36ec84ec1e12c
2024-12-09T17:07:48,324 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/.tmp/info/6529bebdb7c44fa19aa8f0184bccff15 is 45, key is default/info:d/1733764023181/Put/seqid=0
2024-12-09T17:07:48,328 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/info/3b1f8b49b7654eb283d36ec84ec1e12c, entries=20, sequenceid=16, filesize=8.1 K
2024-12-09T17:07:48,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741848_1033 (size=5037)
2024-12-09T17:07:48,330 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/.tmp/table/cf5f1567b68243139b1df0fb0aa78def as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/table/cf5f1567b68243139b1df0fb0aa78def
2024-12-09T17:07:48,331 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/.tmp/info/6529bebdb7c44fa19aa8f0184bccff15
2024-12-09T17:07:48,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741848_1033 (size=5037)
2024-12-09T17:07:48,337 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/table/cf5f1567b68243139b1df0fb0aa78def, entries=4, sequenceid=16, filesize=5.4 K
2024-12-09T17:07:48,337 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/.tmp/info/6529bebdb7c44fa19aa8f0184bccff15 as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/info/6529bebdb7c44fa19aa8f0184bccff15
2024-12-09T17:07:48,338 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.90 KB/2972, heapSize ~5.14 KB/5264, currentSize=0 B/0 for 1588230740 in 71ms, sequenceid=16, compaction requested=false
2024-12-09T17:07:48,342 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/meta/1588230740/recovered.edits/19.seqid, newMaxSeqId=19, maxSeqId=1
2024-12-09T17:07:48,343 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-09T17:07:48,343 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:07:48,343 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:07:48,343 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-09T17:07:48,344 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/info/6529bebdb7c44fa19aa8f0184bccff15, entries=2, sequenceid=8, filesize=4.9 K
2024-12-09T17:07:48,345 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 4b4ce617a32ce6991d195b2772ff6e63 in 36ms, sequenceid=8, compaction requested=false
2024-12-09T17:07:48,351 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/data/hbase/namespace/4b4ce617a32ce6991d195b2772ff6e63/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-09T17:07:48,351 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:48,351 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4b4ce617a32ce6991d195b2772ff6e63:

2024-12-09T17:07:48,352 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733764022653.4b4ce617a32ce6991d195b2772ff6e63.
2024-12-09T17:07:48,467 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,36387,1733764021607; all regions closed.
2024-12-09T17:07:48,468 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607
2024-12-09T17:07:48,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741844_1028 (size=761)
2024-12-09T17:07:48,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741844_1028 (size=761)
2024-12-09T17:07:48,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:48,978 INFO  [regionserver/80c69eb3c456:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-09T17:07:48,979 INFO  [regionserver/80c69eb3c456:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-09T17:07:49,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:49,979 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:07:50,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:51,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-09T17:07:51,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:52,208 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta after 4001ms
2024-12-09T17:07:52,209 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta to hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/oldWALs/80c69eb3c456%2C36387%2C1733764021607.meta.1733764022567.meta
2024-12-09T17:07:52,216 DEBUG [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/oldWALs
2024-12-09T17:07:52,216 INFO  [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C36387%2C1733764021607.meta:.meta(num 1733764068197)
2024-12-09T17:07:52,217 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/WALs/80c69eb3c456,36387,1733764021607
2024-12-09T17:07:52,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741843_1027 (size=1979)
2024-12-09T17:07:52,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741843_1027 (size=1979)
2024-12-09T17:07:52,227 DEBUG [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(1071): Moved 4 WAL file(s) to /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/oldWALs
2024-12-09T17:07:52,227 INFO  [RS:0;80c69eb3c456:36387 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C36387%2C1733764021607:(num 1733764068191)
2024-12-09T17:07:52,227 DEBUG [RS:0;80c69eb3c456:36387 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:07:52,227 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:07:52,227 INFO  [RS:0;80c69eb3c456:36387 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-09T17:07:52,227 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:07:52,228 INFO  [RS:0;80c69eb3c456:36387 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36387
2024-12-09T17:07:52,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:07:52,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,36387,1733764021607
2024-12-09T17:07:52,278 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,36387,1733764021607]
2024-12-09T17:07:52,278 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,36387,1733764021607; numProcessing=1
2024-12-09T17:07:52,287 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,36387,1733764021607 already deleted, retry=false
2024-12-09T17:07:52,287 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,36387,1733764021607 expired; onlineServers=0
2024-12-09T17:07:52,287 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,41375,1733764021439' *****
2024-12-09T17:07:52,287 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-09T17:07:52,287 DEBUG [M:0;80c69eb3c456:41375 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@474bc74b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:07:52,287 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,41375,1733764021439
2024-12-09T17:07:52,287 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,41375,1733764021439; all regions closed.
2024-12-09T17:07:52,287 DEBUG [M:0;80c69eb3c456:41375 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:07:52,288 DEBUG [M:0;80c69eb3c456:41375 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-09T17:07:52,288 DEBUG [M:0;80c69eb3c456:41375 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-09T17:07:52,288 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-09T17:07:52,288 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764021909 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764021909,5,FailOnTimeoutGroup]
2024-12-09T17:07:52,288 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764021909 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764021909,5,FailOnTimeoutGroup]
2024-12-09T17:07:52,288 INFO  [M:0;80c69eb3c456:41375 {}] hbase.ChoreService(370): Chore service for: master/80c69eb3c456:0 had [] on shutdown
2024-12-09T17:07:52,288 DEBUG [M:0;80c69eb3c456:41375 {}] master.HMaster(1733): Stopping service threads
2024-12-09T17:07:52,288 INFO  [M:0;80c69eb3c456:41375 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-09T17:07:52,289 INFO  [M:0;80c69eb3c456:41375 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-09T17:07:52,289 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-09T17:07:52,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-09T17:07:52,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:52,295 DEBUG [M:0;80c69eb3c456:41375 {}] zookeeper.ZKUtil(347): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-09T17:07:52,295 WARN  [M:0;80c69eb3c456:41375 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-09T17:07:52,295 INFO  [M:0;80c69eb3c456:41375 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-09T17:07:52,296 INFO  [M:0;80c69eb3c456:41375 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-09T17:07:52,296 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:07:52,296 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:52,296 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:52,296 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:07:52,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:07:52,296 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:52,296 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.09 KB heapSize=49.26 KB
2024-12-09T17:07:52,319 DEBUG [M:0;80c69eb3c456:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbf7b9a320334067bc5cd7c4048deed3 is 82, key is hbase:meta,,1/info:regioninfo/1733764022592/Put/seqid=0
2024-12-09T17:07:52,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741849_1034 (size=5672)
2024-12-09T17:07:52,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741849_1034 (size=5672)
2024-12-09T17:07:52,324 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbf7b9a320334067bc5cd7c4048deed3
2024-12-09T17:07:52,337 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1016: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-09T17:07:52,342 DEBUG [M:0;80c69eb3c456:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17e7aaee8f86452dbdd20e46c26a8b7d is 778, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733764023794/Put/seqid=0
2024-12-09T17:07:52,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741850_1035 (size=7469)
2024-12-09T17:07:52,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741850_1035 (size=7469)
2024-12-09T17:07:52,348 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17e7aaee8f86452dbdd20e46c26a8b7d
2024-12-09T17:07:52,366 DEBUG [M:0;80c69eb3c456:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d5dbc86c61c428593b338c3b502d072 is 69, key is 80c69eb3c456,36387,1733764021607/rs:state/1733764021959/Put/seqid=0
2024-12-09T17:07:52,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741851_1036 (size=5156)
2024-12-09T17:07:52,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741851_1036 (size=5156)
2024-12-09T17:07:52,371 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d5dbc86c61c428593b338c3b502d072
2024-12-09T17:07:52,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:07:52,379 INFO  [RS:0;80c69eb3c456:36387 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,36387,1733764021607; zookeeper connection closed.
2024-12-09T17:07:52,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36387-0x1000bc2992f0001, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:07:52,379 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28f3c95c {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28f3c95c
2024-12-09T17:07:52,379 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-09T17:07:52,391 DEBUG [M:0;80c69eb3c456:41375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5cda9aa242314db98b5d034b2f1296b2 is 52, key is load_balancer_on/state:d/1733764023346/Put/seqid=0
2024-12-09T17:07:52,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741852_1037 (size=5056)
2024-12-09T17:07:52,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741852_1037 (size=5056)
2024-12-09T17:07:52,397 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5cda9aa242314db98b5d034b2f1296b2
2024-12-09T17:07:52,401 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbf7b9a320334067bc5cd7c4048deed3 as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbf7b9a320334067bc5cd7c4048deed3
2024-12-09T17:07:52,407 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbf7b9a320334067bc5cd7c4048deed3, entries=8, sequenceid=96, filesize=5.5 K
2024-12-09T17:07:52,408 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17e7aaee8f86452dbdd20e46c26a8b7d as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/17e7aaee8f86452dbdd20e46c26a8b7d
2024-12-09T17:07:52,413 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/17e7aaee8f86452dbdd20e46c26a8b7d, entries=11, sequenceid=96, filesize=7.3 K
2024-12-09T17:07:52,414 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d5dbc86c61c428593b338c3b502d072 as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d5dbc86c61c428593b338c3b502d072
2024-12-09T17:07:52,419 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d5dbc86c61c428593b338c3b502d072, entries=1, sequenceid=96, filesize=5.0 K
2024-12-09T17:07:52,420 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5cda9aa242314db98b5d034b2f1296b2 as hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5cda9aa242314db98b5d034b2f1296b2
2024-12-09T17:07:52,425 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37483/user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5cda9aa242314db98b5d034b2f1296b2, entries=1, sequenceid=96, filesize=4.9 K
2024-12-09T17:07:52,426 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.09 KB/41052, heapSize ~49.20 KB/50376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 130ms, sequenceid=96, compaction requested=false
2024-12-09T17:07:52,427 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:52,427 DEBUG [M:0;80c69eb3c456:41375 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:07:52,428 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/649b5bd7-2c44-d718-110a-9c7218c75693/MasterData/WALs/80c69eb3c456,41375,1733764021439
2024-12-09T17:07:52,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39595 is added to blk_1073741840_1021 (size=757)
2024-12-09T17:07:52,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34301 is added to blk_1073741840_1021 (size=757)
2024-12-09T17:07:52,430 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:07:52,430 INFO  [M:0;80c69eb3c456:41375 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-09T17:07:52,430 INFO  [M:0;80c69eb3c456:41375 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41375
2024-12-09T17:07:52,436 DEBUG [M:0;80c69eb3c456:41375 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/80c69eb3c456,41375,1733764021439 already deleted, retry=false
2024-12-09T17:07:52,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:07:52,545 INFO  [M:0;80c69eb3c456:41375 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,41375,1733764021439; zookeeper connection closed.
2024-12-09T17:07:52,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41375-0x1000bc2992f0000, quorum=127.0.0.1:51082, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:07:52,548 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@15c82a5e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:52,548 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6bfee350{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:52,548 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:52,548 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2dc68e60{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:52,549 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@148b9396{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:52,551 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:07:52,551 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:07:52,551 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495047503-172.17.0.2-1733764019933 (Datanode Uuid d1e7194a-829c-46db-a47a-b36801bd8363) service to localhost/127.0.0.1:37483
2024-12-09T17:07:52,551 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:07:52,551 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data3/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:52,552 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data4/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:52,552 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:07:52,554 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@505145a5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:52,555 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16267e26{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:52,555 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:52,555 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@bb3bbf0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:52,555 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef7e454{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:52,557 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:07:52,557 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:07:52,557 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:07:52,557 WARN  [BP-495047503-172.17.0.2-1733764019933 heartbeating to localhost/127.0.0.1:37483 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495047503-172.17.0.2-1733764019933 (Datanode Uuid 299775b1-6b47-45a6-8426-21a4db1c3ca7) service to localhost/127.0.0.1:37483
2024-12-09T17:07:52,558 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data1/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:52,558 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/cluster_2e61575d-c0db-0f8e-24e6-28fec8dc0845/dfs/data/data2/current/BP-495047503-172.17.0.2-1733764019933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:07:52,558 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:07:52,565 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ecc9f79{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:07:52,565 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@708ba1ca{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:07:52,565 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:07:52,565 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68473501{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:07:52,566 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@140f7a56{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir/,STOPPED}
2024-12-09T17:07:52,573 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-09T17:07:52,589 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-09T17:07:52,594 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=101 (was 88)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37483
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37483 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RS-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:37483
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37483
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37483 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37483 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37483
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37483
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 424) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=29 (was 50), ProcessCount=11 (was 11), AvailableMemoryMB=4951 (was 5129)
2024-12-09T17:07:52,599 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=101, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=29, ProcessCount=11, AvailableMemoryMB=4951
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.log.dir so I do NOT create it in target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d255fc5f-d0f6-a00d-8582-f37a64bb7775/hadoop.tmp.dir so I do NOT create it in target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da, deleteOnExit=true
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/test.cache.data in system properties and HBase conf
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.tmp.dir in system properties and HBase conf
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir in system properties and HBase conf
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-09T17:07:52,600 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-09T17:07:52,601 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-09T17:07:52,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/nfs.dump.dir in system properties and HBase conf
2024-12-09T17:07:52,602 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/java.io.tmpdir in system properties and HBase conf
2024-12-09T17:07:52,602 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:07:52,602 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-09T17:07:52,602 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-09T17:07:52,613 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:07:52,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:52,904 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:52,907 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:52,909 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:52,909 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:52,909 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:07:52,909 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:52,910 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22a9191b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:52,910 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@262a1159{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:53,001 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58ac6791{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/java.io.tmpdir/jetty-localhost-43801-hadoop-hdfs-3_4_1-tests_jar-_-any-6005991885046450026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:07:53,001 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c7be892{HTTP/1.1, (http/1.1)}{localhost:43801}
2024-12-09T17:07:53,001 INFO  [Time-limited test {}] server.Server(415): Started @227421ms
2024-12-09T17:07:53,012 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:07:53,203 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:53,205 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:53,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:53,206 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:53,206 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:07:53,206 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7145ab8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:53,207 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36119ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:53,294 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b4b904e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/java.io.tmpdir/jetty-localhost-38609-hadoop-hdfs-3_4_1-tests_jar-_-any-5019561590994966306/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:53,294 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@50f9eea5{HTTP/1.1, (http/1.1)}{localhost:38609}
2024-12-09T17:07:53,294 INFO  [Time-limited test {}] server.Server(415): Started @227714ms
2024-12-09T17:07:53,295 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:53,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,319 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,319 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,319 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,320 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,323 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,323 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,323 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,323 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,336 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:07:53,339 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,339 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,339 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,339 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,352 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,352 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,352 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,353 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,353 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,353 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,356 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,356 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,356 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,358 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:07:53,364 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:07:53,367 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:07:53,369 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:07:53,369 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:07:53,369 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:07:53,370 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1611ccec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:07:53,371 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@541bc67d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:07:53,460 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@61ea8370{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/java.io.tmpdir/jetty-localhost-44379-hadoop-hdfs-3_4_1-tests_jar-_-any-4230192493615173418/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:07:53,460 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2959bd10{HTTP/1.1, (http/1.1)}{localhost:44379}
2024-12-09T17:07:53,460 INFO  [Time-limited test {}] server.Server(415): Started @227880ms
2024-12-09T17:07:53,462 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:07:53,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:54,118 WARN  [Thread-1412 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data1/current/BP-850976073-172.17.0.2-1733764072624/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:54,119 WARN  [Thread-1413 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data2/current/BP-850976073-172.17.0.2-1733764072624/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:54,136 WARN  [Thread-1375 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:54,138 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c7c08b362b7164f with lease ID 0x3c91819f68ff515: Processing first storage report for DS-745600da-8fb1-4d02-86e5-0da605258cc2 from datanode DatanodeRegistration(127.0.0.1:43119, datanodeUuid=561690cb-6d19-4005-b2af-ac2472793b59, infoPort=35731, infoSecurePort=0, ipcPort=46135, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624)
2024-12-09T17:07:54,139 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c7c08b362b7164f with lease ID 0x3c91819f68ff515: from storage DS-745600da-8fb1-4d02-86e5-0da605258cc2 node DatanodeRegistration(127.0.0.1:43119, datanodeUuid=561690cb-6d19-4005-b2af-ac2472793b59, infoPort=35731, infoSecurePort=0, ipcPort=46135, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:54,139 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c7c08b362b7164f with lease ID 0x3c91819f68ff515: Processing first storage report for DS-7506d23c-cc38-4dd1-8830-568a2158da2c from datanode DatanodeRegistration(127.0.0.1:43119, datanodeUuid=561690cb-6d19-4005-b2af-ac2472793b59, infoPort=35731, infoSecurePort=0, ipcPort=46135, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624)
2024-12-09T17:07:54,139 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c7c08b362b7164f with lease ID 0x3c91819f68ff515: from storage DS-7506d23c-cc38-4dd1-8830-568a2158da2c node DatanodeRegistration(127.0.0.1:43119, datanodeUuid=561690cb-6d19-4005-b2af-ac2472793b59, infoPort=35731, infoSecurePort=0, ipcPort=46135, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:54,209 WARN  [Thread-1423 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data3/current/BP-850976073-172.17.0.2-1733764072624/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:54,210 WARN  [Thread-1424 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data4/current/BP-850976073-172.17.0.2-1733764072624/current, will proceed with Du for space computation calculation, 
2024-12-09T17:07:54,226 WARN  [Thread-1399 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:07:54,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8619ba5e3914548 with lease ID 0x3c91819f68ff516: Processing first storage report for DS-81f2aacb-a2e5-4c75-87f5-4f3ed589b4f4 from datanode DatanodeRegistration(127.0.0.1:45869, datanodeUuid=04db581a-212a-453d-aa85-af8de0141933, infoPort=43301, infoSecurePort=0, ipcPort=45249, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624)
2024-12-09T17:07:54,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8619ba5e3914548 with lease ID 0x3c91819f68ff516: from storage DS-81f2aacb-a2e5-4c75-87f5-4f3ed589b4f4 node DatanodeRegistration(127.0.0.1:45869, datanodeUuid=04db581a-212a-453d-aa85-af8de0141933, infoPort=43301, infoSecurePort=0, ipcPort=45249, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:54,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe8619ba5e3914548 with lease ID 0x3c91819f68ff516: Processing first storage report for DS-764bfacd-2e79-4305-bd16-276c570fe36f from datanode DatanodeRegistration(127.0.0.1:45869, datanodeUuid=04db581a-212a-453d-aa85-af8de0141933, infoPort=43301, infoSecurePort=0, ipcPort=45249, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624)
2024-12-09T17:07:54,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8619ba5e3914548 with lease ID 0x3c91819f68ff516: from storage DS-764bfacd-2e79-4305-bd16-276c570fe36f node DatanodeRegistration(127.0.0.1:45869, datanodeUuid=04db581a-212a-453d-aa85-af8de0141933, infoPort=43301, infoSecurePort=0, ipcPort=45249, storageInfo=lv=-57;cid=testClusterID;nsid=283480189;c=1733764072624), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:07:54,290 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336
2024-12-09T17:07:54,311 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/zookeeper_0, clientPort=53522, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-09T17:07:54,313 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53522
2024-12-09T17:07:54,313 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,316 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:07:54,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:07:54,327 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e with version=8
2024-12-09T17:07:54,327 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase-staging
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:07:54,329 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:07:54,330 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36389
2024-12-09T17:07:54,330 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,332 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,334 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36389 connecting to ZooKeeper ensemble=127.0.0.1:53522
2024-12-09T17:07:54,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:363890x0, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:07:54,383 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36389-0x1000bc367c70000 connected
2024-12-09T17:07:54,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:07:54,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:07:54,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:07:54,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36389
2024-12-09T17:07:54,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36389
2024-12-09T17:07:54,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36389
2024-12-09T17:07:54,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36389
2024-12-09T17:07:54,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36389
2024-12-09T17:07:54,448 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e, hbase.cluster.distributed=false
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:07:54,462 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:07:54,463 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46369
2024-12-09T17:07:54,463 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:07:54,464 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:07:54,464 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,466 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,468 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:46369 connecting to ZooKeeper ensemble=127.0.0.1:53522
2024-12-09T17:07:54,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463690x0, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:07:54,479 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:463690x0, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:07:54,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46369-0x1000bc367c70001 connected
2024-12-09T17:07:54,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:07:54,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:07:54,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46369
2024-12-09T17:07:54,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46369
2024-12-09T17:07:54,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46369
2024-12-09T17:07:54,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46369
2024-12-09T17:07:54,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46369
2024-12-09T17:07:54,482 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:54,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:54,487 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,491 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80c69eb3c456:36389
2024-12-09T17:07:54,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:07:54,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:07:54,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,495 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:07:54,496 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80c69eb3c456,36389,1733764074329 from backup master directory
2024-12-09T17:07:54,496 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:07:54,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:54,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:07:54,503 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:07:54,503 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:07:54,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:07:54,514 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/hbase.id with ID: 589aa713-1c68-4f55-85c3-db52bfc8b6a2
2024-12-09T17:07:54,525 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:07:54,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:07:54,544 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:07:54,545 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-09T17:07:54,545 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:07:54,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:07:54,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:07:54,552 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store
2024-12-09T17:07:54,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:07:54,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:07:54,560 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:54,560 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:07:54,560 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:54,560 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:54,560 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:07:54,560 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:54,560 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:07:54,560 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:07:54,561 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/.initializing
2024-12-09T17:07:54,561 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/WALs/80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,563 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C36389%2C1733764074329, suffix=, logDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/WALs/80c69eb3c456,36389,1733764074329, archiveDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/oldWALs, maxLogs=10
2024-12-09T17:07:54,563 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C36389%2C1733764074329.1733764074563
2024-12-09T17:07:54,570 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/WALs/80c69eb3c456,36389,1733764074329/80c69eb3c456%2C36389%2C1733764074329.1733764074563
2024-12-09T17:07:54,571 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43301:43301),(127.0.0.1/127.0.0.1:35731:35731)]
2024-12-09T17:07:54,571 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:54,571 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:54,571 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:54,571 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:54,572 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:54,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-09T17:07:54,574 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:54,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:54,575 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-09T17:07:54,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:54,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:54,577 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-09T17:07:54,577 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,577 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:54,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:07:54,579 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-09T17:07:54,579 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,579 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:54,580 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:54,581 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:54,583 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-09T17:07:54,584 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:07:54,586 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:54,586 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=854582, jitterRate=0.08665788173675537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-09T17:07:54,587 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:07:54,587 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-09T17:07:54,590 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd4aebb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:07:54,591 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-09T17:07:54,591 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-09T17:07:54,591 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-09T17:07:54,591 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-09T17:07:54,592 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-09T17:07:54,592 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-09T17:07:54,592 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-09T17:07:54,593 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-09T17:07:54,594 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-09T17:07:54,603 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-09T17:07:54,603 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-09T17:07:54,604 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-09T17:07:54,611 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-09T17:07:54,612 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-09T17:07:54,612 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-09T17:07:54,620 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-09T17:07:54,620 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-09T17:07:54,628 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-09T17:07:54,630 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-09T17:07:54,636 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-09T17:07:54,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:07:54,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:07:54,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,646 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=80c69eb3c456,36389,1733764074329, sessionid=0x1000bc367c70000, setting cluster-up flag (Was=false)
2024-12-09T17:07:54,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,687 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-09T17:07:54,688 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:54,728 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-09T17:07:54,730 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,36389,1733764074329
2024-12-09T17:07:54,733 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-09T17:07:54,733 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-09T17:07:54,734 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-09T17:07:54,734 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80c69eb3c456,36389,1733764074329
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-09T17:07:54,734 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:54,734 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:54,734 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:54,734 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:07:54,734 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80c69eb3c456:0, corePoolSize=10, maxPoolSize=10
2024-12-09T17:07:54,735 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,735 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:07:54,735 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,736 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733764104736
2024-12-09T17:07:54,736 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-09T17:07:54,736 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-09T17:07:54,736 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-09T17:07:54,736 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-09T17:07:54,737 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-09T17:07:54,737 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-09T17:07:54,737 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:07:54,737 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-09T17:07:54,737 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,737 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-09T17:07:54,737 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-09T17:07:54,738 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-09T17:07:54,738 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-09T17:07:54,738 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-09T17:07:54,738 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,739 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:07:54,739 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764074738,5,FailOnTimeoutGroup]
2024-12-09T17:07:54,739 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764074739,5,FailOnTimeoutGroup]
2024-12-09T17:07:54,739 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,740 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-09T17:07:54,740 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,740 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,748 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:07:54,748 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:07:54,749 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-09T17:07:54,750 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e
2024-12-09T17:07:54,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:07:54,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:07:54,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:54,758 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:07:54,759 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:07:54,759 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:54,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:07:54,761 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:07:54,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,762 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:54,762 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:07:54,763 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:07:54,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:54,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:54,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740
2024-12-09T17:07:54,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740
2024-12-09T17:07:54,767 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:07:54,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:07:54,771 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:54,771 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=879945, jitterRate=0.11890880763530731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:07:54,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:07:54,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:07:54,771 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:07:54,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:07:54,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:07:54,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:07:54,772 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:07:54,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:07:54,773 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:07:54,773 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-09T17:07:54,773 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-09T17:07:54,774 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-09T17:07:54,775 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-09T17:07:54,792 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80c69eb3c456:46369
2024-12-09T17:07:54,793 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1008): ClusterId : 589aa713-1c68-4f55-85c3-db52bfc8b6a2
2024-12-09T17:07:54,793 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:07:54,804 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:07:54,804 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:07:54,813 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:07:54,813 DEBUG [RS:0;80c69eb3c456:46369 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac32a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:07:54,813 DEBUG [RS:0;80c69eb3c456:46369 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6c41b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:07:54,813 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:07:54,813 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:07:54,813 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:07:54,814 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,36389,1733764074329 with isa=80c69eb3c456/172.17.0.2:46369, startcode=1733764074461
2024-12-09T17:07:54,814 DEBUG [RS:0;80c69eb3c456:46369 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:07:54,816 INFO  [RS-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34919, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:07:54,817 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36389 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,46369,1733764074461
2024-12-09T17:07:54,817 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36389 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:54,818 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e
2024-12-09T17:07:54,818 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:43331
2024-12-09T17:07:54,818 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:07:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:07:54,829 DEBUG [RS:0;80c69eb3c456:46369 {}] zookeeper.ZKUtil(111): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,46369,1733764074461
2024-12-09T17:07:54,829 WARN  [RS:0;80c69eb3c456:46369 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:07:54,829 INFO  [RS:0;80c69eb3c456:46369 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:07:54,829 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461
2024-12-09T17:07:54,830 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,46369,1733764074461]
2024-12-09T17:07:54,834 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:07:54,835 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:07:54,837 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:07:54,838 INFO  [RS:0;80c69eb3c456:46369 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:07:54,838 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,838 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:07:54,840 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,840 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,841 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:07:54,841 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:07:54,841 DEBUG [RS:0;80c69eb3c456:46369 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:07:54,841 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,842 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,842 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,842 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,842 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46369,1733764074461-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:07:54,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:54,855 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:07:54,856 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46369,1733764074461-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:54,866 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.Replication(204): 80c69eb3c456,46369,1733764074461 started
2024-12-09T17:07:54,866 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,46369,1733764074461, RpcServer on 80c69eb3c456/172.17.0.2:46369, sessionid=0x1000bc367c70001
2024-12-09T17:07:54,866 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:07:54,866 DEBUG [RS:0;80c69eb3c456:46369 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,46369,1733764074461
2024-12-09T17:07:54,866 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,46369,1733764074461'
2024-12-09T17:07:54,866 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:07:54,867 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:07:54,867 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:07:54,867 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:07:54,867 DEBUG [RS:0;80c69eb3c456:46369 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,46369,1733764074461
2024-12-09T17:07:54,867 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,46369,1733764074461'
2024-12-09T17:07:54,867 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:07:54,868 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:07:54,868 DEBUG [RS:0;80c69eb3c456:46369 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:07:54,868 INFO  [RS:0;80c69eb3c456:46369 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:07:54,868 INFO  [RS:0;80c69eb3c456:46369 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:07:54,925 WARN  [80c69eb3c456:36389 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-09T17:07:54,970 INFO  [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C46369%2C1733764074461, suffix=, logDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461, archiveDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/oldWALs, maxLogs=32
2024-12-09T17:07:54,972 INFO  [RS:0;80c69eb3c456:46369 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C46369%2C1733764074461.1733764074972
2024-12-09T17:07:54,982 INFO  [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764074972
2024-12-09T17:07:54,982 DEBUG [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731),(127.0.0.1/127.0.0.1:43301:43301)]
2024-12-09T17:07:55,176 DEBUG [80c69eb3c456:36389 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-09T17:07:55,176 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:55,178 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,46369,1733764074461, state=OPENING
2024-12-09T17:07:55,220 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-09T17:07:55,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:55,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:55,230 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=80c69eb3c456,46369,1733764074461}]
2024-12-09T17:07:55,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:55,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:55,385 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:07:55,386 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-09T17:07:55,392 INFO  [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51380, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-09T17:07:55,396 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-09T17:07:55,396 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:07:55,399 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C46369%2C1733764074461.meta, suffix=.meta, logDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461, archiveDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/oldWALs, maxLogs=32
2024-12-09T17:07:55,399 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C46369%2C1733764074461.meta.1733764075399.meta
2024-12-09T17:07:55,412 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.meta.1733764075399.meta
2024-12-09T17:07:55,412 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731),(127.0.0.1/127.0.0.1:43301:43301)]
2024-12-09T17:07:55,412 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:55,413 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-09T17:07:55,413 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-09T17:07:55,413 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-09T17:07:55,413 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-09T17:07:55,413 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:55,413 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-09T17:07:55,413 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-09T17:07:55,414 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:07:55,415 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:07:55,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:55,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:55,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:07:55,416 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:07:55,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:55,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:55,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:07:55,417 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:07:55,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:55,418 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:07:55,419 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740
2024-12-09T17:07:55,420 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740
2024-12-09T17:07:55,421 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:07:55,422 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:07:55,423 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=742926, jitterRate=-0.055321529507637024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:07:55,423 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:07:55,424 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733764075385
2024-12-09T17:07:55,426 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-09T17:07:55,426 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-09T17:07:55,426 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:55,427 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,46369,1733764074461, state=OPEN
2024-12-09T17:07:55,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:07:55,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:07:55,458 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:55,458 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:07:55,461 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-09T17:07:55,461 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=80c69eb3c456,46369,1733764074461 in 228 msec
2024-12-09T17:07:55,463 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-09T17:07:55,463 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 688 msec
2024-12-09T17:07:55,466 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 732 msec
2024-12-09T17:07:55,466 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733764075466, completionTime=-1
2024-12-09T17:07:55,466 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-09T17:07:55,466 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-09T17:07:55,467 DEBUG [hconnection-0x20b7d63f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:07:55,468 INFO  [RS-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51392, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:07:55,469 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-09T17:07:55,469 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733764135469
2024-12-09T17:07:55,469 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733764195469
2024-12-09T17:07:55,469 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36389,1733764074329-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36389,1733764074329-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36389,1733764074329-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80c69eb3c456:36389, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-09T17:07:55,495 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:07:55,497 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-09T17:07:55,497 DEBUG [master/80c69eb3c456:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-09T17:07:55,498 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:07:55,498 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:55,499 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:07:55,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:07:55,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:07:55,509 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5bcbb40859410c4475bd141ae8db5938, NAME => 'hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e
2024-12-09T17:07:55,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:07:55,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:07:55,518 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:55,518 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 5bcbb40859410c4475bd141ae8db5938, disabling compactions & flushes
2024-12-09T17:07:55,518 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,518 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,518 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938. after waiting 0 ms
2024-12-09T17:07:55,518 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,518 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,518 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5bcbb40859410c4475bd141ae8db5938:

2024-12-09T17:07:55,520 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:07:55,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733764075520"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764075520"}]},"ts":"1733764075520"}
2024-12-09T17:07:55,522 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:07:55,523 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:07:55,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764075523"}]},"ts":"1733764075523"}
2024-12-09T17:07:55,525 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-09T17:07:55,545 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5bcbb40859410c4475bd141ae8db5938, ASSIGN}]
2024-12-09T17:07:55,546 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5bcbb40859410c4475bd141ae8db5938, ASSIGN
2024-12-09T17:07:55,547 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=5bcbb40859410c4475bd141ae8db5938, ASSIGN; state=OFFLINE, location=80c69eb3c456,46369,1733764074461; forceNewPlan=false, retain=false
2024-12-09T17:07:55,698 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5bcbb40859410c4475bd141ae8db5938, regionState=OPENING, regionLocation=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:55,699 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 5bcbb40859410c4475bd141ae8db5938, server=80c69eb3c456,46369,1733764074461}]
2024-12-09T17:07:55,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:55,852 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:07:55,860 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 5bcbb40859410c4475bd141ae8db5938, NAME => 'hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:55,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:07:55,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:55,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:07:55,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:07:55,864 INFO  [StoreOpener-5bcbb40859410c4475bd141ae8db5938-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5bcbb40859410c4475bd141ae8db5938 
2024-12-09T17:07:55,866 INFO  [StoreOpener-5bcbb40859410c4475bd141ae8db5938-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bcbb40859410c4475bd141ae8db5938 columnFamilyName info
2024-12-09T17:07:55,866 DEBUG [StoreOpener-5bcbb40859410c4475bd141ae8db5938-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:55,867 INFO  [StoreOpener-5bcbb40859410c4475bd141ae8db5938-1 {}] regionserver.HStore(327): Store=5bcbb40859410c4475bd141ae8db5938/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:55,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:07:55,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:07:55,872 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:07:55,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:55,875 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 5bcbb40859410c4475bd141ae8db5938; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=696931, jitterRate=-0.11380665004253387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:07:55,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 5bcbb40859410c4475bd141ae8db5938:

2024-12-09T17:07:55,876 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938., pid=6, masterSystemTime=1733764075852
2024-12-09T17:07:55,878 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,878 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:07:55,879 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5bcbb40859410c4475bd141ae8db5938, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:55,883 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-09T17:07:55,884 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 5bcbb40859410c4475bd141ae8db5938, server=80c69eb3c456,46369,1733764074461 in 182 msec
2024-12-09T17:07:55,885 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-09T17:07:55,885 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=5bcbb40859410c4475bd141ae8db5938, ASSIGN in 338 msec
2024-12-09T17:07:55,886 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:07:55,886 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764075886"}]},"ts":"1733764075886"}
2024-12-09T17:07:55,887 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-09T17:07:55,896 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:07:55,897 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 401 msec
2024-12-09T17:07:55,898 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-09T17:07:55,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:07:55,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:55,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:07:55,907 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-09T17:07:55,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:07:55,931 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 23 msec
2024-12-09T17:07:55,940 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-09T17:07:55,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:07:55,973 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 33 msec
2024-12-09T17:07:56,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-09T17:07:56,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.517sec
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-09T17:07:56,021 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36389,1733764074329-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:07:56,022 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36389,1733764074329-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-09T17:07:56,026 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-09T17:07:56,026 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-09T17:07:56,026 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,36389,1733764074329-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:07:56,086 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58c48638 to 127.0.0.1:53522 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1726c844
2024-12-09T17:07:56,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e2b1db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:07:56,098 DEBUG [hconnection-0x7d6cfa0d-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:07:56,101 INFO  [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51408, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:07:56,103 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=80c69eb3c456,36389,1733764074329
2024-12-09T17:07:56,103 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:07:56,107 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-09T17:07:56,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-09T17:07:56,111 INFO  [RS-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51984, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-09T17:07:56,113 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-09T17:07:56,113 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-09T17:07:56,113 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:07:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:07:56,115 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:07:56,115 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:56,116 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 9
2024-12-09T17:07:56,116 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:07:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:07:56,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741837_1013 (size=405)
2024-12-09T17:07:56,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741837_1013 (size=405)
2024-12-09T17:07:56,125 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2459d518b0bf9e0f10999c1bed6ae018, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e
2024-12-09T17:07:56,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741838_1014 (size=88)
2024-12-09T17:07:56,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741838_1014 (size=88)
2024-12-09T17:07:56,132 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:56,132 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1681): Closing 2459d518b0bf9e0f10999c1bed6ae018, disabling compactions & flushes
2024-12-09T17:07:56,132 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,132 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,132 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018. after waiting 0 ms
2024-12-09T17:07:56,132 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,132 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,132 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:07:56,134 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:07:56,134 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733764076134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764076134"}]},"ts":"1733764076134"}
2024-12-09T17:07:56,136 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:07:56,137 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:07:56,137 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764076137"}]},"ts":"1733764076137"}
2024-12-09T17:07:56,139 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-09T17:07:56,153 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=2459d518b0bf9e0f10999c1bed6ae018, ASSIGN}]
2024-12-09T17:07:56,155 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=2459d518b0bf9e0f10999c1bed6ae018, ASSIGN
2024-12-09T17:07:56,156 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=2459d518b0bf9e0f10999c1bed6ae018, ASSIGN; state=OFFLINE, location=80c69eb3c456,46369,1733764074461; forceNewPlan=false, retain=false
2024-12-09T17:07:56,306 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=2459d518b0bf9e0f10999c1bed6ae018, regionState=OPENING, regionLocation=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:56,310 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 2459d518b0bf9e0f10999c1bed6ae018, server=80c69eb3c456,46369,1733764074461}]
2024-12-09T17:07:56,465 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:07:56,473 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,473 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 2459d518b0bf9e0f10999c1bed6ae018, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:07:56,473 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:07:56,474 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:07:56,474 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:07:56,474 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:07:56,476 INFO  [StoreOpener-2459d518b0bf9e0f10999c1bed6ae018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2459d518b0bf9e0f10999c1bed6ae018 
2024-12-09T17:07:56,479 INFO  [StoreOpener-2459d518b0bf9e0f10999c1bed6ae018-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2459d518b0bf9e0f10999c1bed6ae018 columnFamilyName info
2024-12-09T17:07:56,479 DEBUG [StoreOpener-2459d518b0bf9e0f10999c1bed6ae018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:07:56,480 INFO  [StoreOpener-2459d518b0bf9e0f10999c1bed6ae018-1 {}] regionserver.HStore(327): Store=2459d518b0bf9e0f10999c1bed6ae018/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:07:56,481 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:07:56,482 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:07:56,487 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:07:56,491 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:07:56,492 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 2459d518b0bf9e0f10999c1bed6ae018; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741188, jitterRate=-0.057531535625457764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:07:56,493 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:07:56,494 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018., pid=11, masterSystemTime=1733764076465
2024-12-09T17:07:56,496 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,496 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:07:56,497 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=2459d518b0bf9e0f10999c1bed6ae018, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,46369,1733764074461
2024-12-09T17:07:56,500 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-09T17:07:56,501 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 2459d518b0bf9e0f10999c1bed6ae018, server=80c69eb3c456,46369,1733764074461 in 188 msec
2024-12-09T17:07:56,503 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-09T17:07:56,503 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=2459d518b0bf9e0f10999c1bed6ae018, ASSIGN in 348 msec
2024-12-09T17:07:56,503 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:07:56,504 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764076503"}]},"ts":"1733764076503"}
2024-12-09T17:07:56,505 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-09T17:07:56,512 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:07:56,514 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 400 msec
2024-12-09T17:07:56,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:57,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:58,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:07:59,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:00,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:00,917 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:08:00,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,937 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,937 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,937 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,939 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:08:00,943 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-09T17:08:00,944 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-09T17:08:00,944 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-09T17:08:01,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:01,349 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-09T17:08:01,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:01,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta after 68082ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:08:02,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:03,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:04,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:05,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:08:06,120 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 9 completed
2024-12-09T17:08:06,127 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:06,127 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:06,137 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush hbase:namespace
2024-12-09T17:08:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace
2024-12-09T17:08:06,143 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_PREPARE
2024-12-09T17:08:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12
2024-12-09T17:08:06,180 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-09T17:08:06,182 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-09T17:08:06,343 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:08:06,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46369 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13
2024-12-09T17:08:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:06,346 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 5bcbb40859410c4475bd141ae8db5938 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-09T17:08:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/.tmp/info/8336c3b1f2f2466aa598ba0ad048bdbb is 45, key is default/info:d/1733764075910/Put/seqid=0
2024-12-09T17:08:06,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741839_1015 (size=5037)
2024-12-09T17:08:06,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741839_1015 (size=5037)
2024-12-09T17:08:06,369 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/.tmp/info/8336c3b1f2f2466aa598ba0ad048bdbb
2024-12-09T17:08:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/.tmp/info/8336c3b1f2f2466aa598ba0ad048bdbb as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/info/8336c3b1f2f2466aa598ba0ad048bdbb
2024-12-09T17:08:06,382 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/info/8336c3b1f2f2466aa598ba0ad048bdbb, entries=2, sequenceid=6, filesize=4.9 K
2024-12-09T17:08:06,383 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 5bcbb40859410c4475bd141ae8db5938 in 37ms, sequenceid=6, compaction requested=false
2024-12-09T17:08:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 5bcbb40859410c4475bd141ae8db5938:

2024-12-09T17:08:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13
2024-12-09T17:08:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster(4106): Remote procedure done, pid=13
2024-12-09T17:08:06,389 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12
2024-12-09T17:08:06,389 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 205 msec
2024-12-09T17:08:06,391 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace in 251 msec
2024-12-09T17:08:06,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:07,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:08,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:09,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:10,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:11,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:12,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:13,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:14,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:15,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12
2024-12-09T17:08:16,147 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: hbase:namespace, procId: 12 completed
2024-12-09T17:08:16,159 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14
2024-12-09T17:08:16,161 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-09T17:08:16,162 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-09T17:08:16,162 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-09T17:08:16,313 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:08:16,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46369 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15
2024-12-09T17:08:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:16,315 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing 2459d518b0bf9e0f10999c1bed6ae018 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-09T17:08:16,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/0ec86a0a6661424badce1b77753ff463 is 1080, key is row0001/info:/1733764096155/Put/seqid=0
2024-12-09T17:08:16,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741840_1016 (size=6033)
2024-12-09T17:08:16,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741840_1016 (size=6033)
2024-12-09T17:08:16,339 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/0ec86a0a6661424badce1b77753ff463
2024-12-09T17:08:16,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/0ec86a0a6661424badce1b77753ff463 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/0ec86a0a6661424badce1b77753ff463
2024-12-09T17:08:16,351 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/0ec86a0a6661424badce1b77753ff463, entries=1, sequenceid=5, filesize=5.9 K
2024-12-09T17:08:16,352 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 2459d518b0bf9e0f10999c1bed6ae018 in 38ms, sequenceid=5, compaction requested=false
2024-12-09T17:08:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:08:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15
2024-12-09T17:08:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster(4106): Remote procedure done, pid=15
2024-12-09T17:08:16,356 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14
2024-12-09T17:08:16,356 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 192 msec
2024-12-09T17:08:16,358 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 198 msec
2024-12-09T17:08:16,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:17,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:18,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:19,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:20,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:21,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:22,875 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:23,877 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:24,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:08:24,878 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:25,814 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:08:25,817 INFO  [RS-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33606, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:08:25,879 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14
2024-12-09T17:08:26,165 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 14 completed
2024-12-09T17:08:26,173 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:26,176 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-09T17:08:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16
2024-12-09T17:08:26,177 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-09T17:08:26,177 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-09T17:08:26,330 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:08:26,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46369 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17
2024-12-09T17:08:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:26,332 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing 2459d518b0bf9e0f10999c1bed6ae018 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-09T17:08:26,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/75f177dc081e4c61bd735fb9e2cdaa9d is 1080, key is row0002/info:/1733764106167/Put/seqid=0
2024-12-09T17:08:26,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741841_1017 (size=6033)
2024-12-09T17:08:26,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741841_1017 (size=6033)
2024-12-09T17:08:26,348 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/75f177dc081e4c61bd735fb9e2cdaa9d
2024-12-09T17:08:26,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/75f177dc081e4c61bd735fb9e2cdaa9d as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/75f177dc081e4c61bd735fb9e2cdaa9d
2024-12-09T17:08:26,360 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/75f177dc081e4c61bd735fb9e2cdaa9d, entries=1, sequenceid=9, filesize=5.9 K
2024-12-09T17:08:26,360 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 2459d518b0bf9e0f10999c1bed6ae018 in 28ms, sequenceid=9, compaction requested=false
2024-12-09T17:08:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:08:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17
2024-12-09T17:08:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster(4106): Remote procedure done, pid=17
2024-12-09T17:08:26,363 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16
2024-12-09T17:08:26,363 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 185 msec
2024-12-09T17:08:26,365 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 191 msec
2024-12-09T17:08:26,880 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:27,882 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:28,883 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:29,884 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:30,886 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:31,887 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:32,888 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:33,890 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:34,891 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:35,892 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16
2024-12-09T17:08:36,177 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 16 completed
2024-12-09T17:08:36,179 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C46369%2C1733764074461.1733764116179
2024-12-09T17:08:36,187 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764074972 with entries=13, filesize=6.41 KB; new WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764116179
2024-12-09T17:08:36,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43301:43301),(127.0.0.1/127.0.0.1:35731:35731)]
2024-12-09T17:08:36,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764074972 is not closed yet, will try archiving it next time
2024-12-09T17:08:36,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741833_1009 (size=6574)
2024-12-09T17:08:36,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741833_1009 (size=6574)
2024-12-09T17:08:36,191 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18
2024-12-09T17:08:36,193 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-09T17:08:36,193 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-09T17:08:36,193 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-09T17:08:36,345 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:08:36,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46369 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19
2024-12-09T17:08:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:36,346 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing 2459d518b0bf9e0f10999c1bed6ae018 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-09T17:08:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/acc6e9a1777c4959a356dcbc21187415 is 1080, key is row0003/info:/1733764116178/Put/seqid=0
2024-12-09T17:08:36,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741843_1019 (size=6033)
2024-12-09T17:08:36,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741843_1019 (size=6033)
2024-12-09T17:08:36,360 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/acc6e9a1777c4959a356dcbc21187415
2024-12-09T17:08:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/acc6e9a1777c4959a356dcbc21187415 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/acc6e9a1777c4959a356dcbc21187415
2024-12-09T17:08:36,376 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/acc6e9a1777c4959a356dcbc21187415, entries=1, sequenceid=13, filesize=5.9 K
2024-12-09T17:08:36,377 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 2459d518b0bf9e0f10999c1bed6ae018 in 31ms, sequenceid=13, compaction requested=true
2024-12-09T17:08:36,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:08:36,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:36,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19
2024-12-09T17:08:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster(4106): Remote procedure done, pid=19
2024-12-09T17:08:36,380 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18
2024-12-09T17:08:36,380 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 186 msec
2024-12-09T17:08:36,382 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-12-09T17:08:36,608 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-09T17:08:36,609 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-09T17:08:36,893 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:37,895 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:38,896 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:39,897 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:40,861 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 5bcbb40859410c4475bd141ae8db5938, had cached 0 bytes from a total of 5037
2024-12-09T17:08:40,898 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:41,474 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 2459d518b0bf9e0f10999c1bed6ae018, had cached 0 bytes from a total of 18099
2024-12-09T17:08:41,900 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:42,901 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:43,902 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:44,903 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:45,904 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18
2024-12-09T17:08:46,195 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 18 completed
2024-12-09T17:08:46,196 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:08:46,199 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:08:46,199 DEBUG [Time-limited test {}] regionserver.HStore(1540): 2459d518b0bf9e0f10999c1bed6ae018/info is initiating minor compaction (all files)
2024-12-09T17:08:46,199 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:08:46,199 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:46,199 INFO  [Time-limited test {}] regionserver.HRegion(2351): Starting compaction of 2459d518b0bf9e0f10999c1bed6ae018/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:46,200 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/0ec86a0a6661424badce1b77753ff463, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/75f177dc081e4c61bd735fb9e2cdaa9d, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/acc6e9a1777c4959a356dcbc21187415] into tmpdir=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp, totalSize=17.7 K
2024-12-09T17:08:46,201 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 0ec86a0a6661424badce1b77753ff463, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733764096155
2024-12-09T17:08:46,202 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 75f177dc081e4c61bd735fb9e2cdaa9d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733764106167
2024-12-09T17:08:46,203 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting acc6e9a1777c4959a356dcbc21187415, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733764116178
2024-12-09T17:08:46,215 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 2459d518b0bf9e0f10999c1bed6ae018#info#compaction#32 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:08:46,216 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/2248a5331bb541fa90281c1a6acaa653 is 1080, key is row0001/info:/1733764096155/Put/seqid=0
2024-12-09T17:08:46,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741844_1020 (size=8296)
2024-12-09T17:08:46,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741844_1020 (size=8296)
2024-12-09T17:08:46,225 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/2248a5331bb541fa90281c1a6acaa653 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/2248a5331bb541fa90281c1a6acaa653
2024-12-09T17:08:46,231 INFO  [Time-limited test {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 2459d518b0bf9e0f10999c1bed6ae018/info of 2459d518b0bf9e0f10999c1bed6ae018 into 2248a5331bb541fa90281c1a6acaa653(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:08:46,231 DEBUG [Time-limited test {}] regionserver.HRegion(2381): Compaction status journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:08:46,233 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C46369%2C1733764074461.1733764126233
2024-12-09T17:08:46,240 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764116179 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764126233
2024-12-09T17:08:46,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43301:43301),(127.0.0.1/127.0.0.1:35731:35731)]
2024-12-09T17:08:46,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764116179 is not closed yet, will try archiving it next time
2024-12-09T17:08:46,240 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764074972 to hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/oldWALs/80c69eb3c456%2C46369%2C1733764074461.1733764074972
2024-12-09T17:08:46,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741842_1018 (size=2520)
2024-12-09T17:08:46,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741842_1018 (size=2520)
2024-12-09T17:08:46,244 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:08:46,245 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-09T17:08:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20
2024-12-09T17:08:46,246 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-09T17:08:46,246 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-09T17:08:46,398 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,46369,1733764074461
2024-12-09T17:08:46,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46369 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21
2024-12-09T17:08:46,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:46,400 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing 2459d518b0bf9e0f10999c1bed6ae018 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-09T17:08:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/7dcc5252983b4e0d99b8e8b65971a174 is 1080, key is row0000/info:/1733764126232/Put/seqid=0
2024-12-09T17:08:46,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741846_1022 (size=6033)
2024-12-09T17:08:46,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741846_1022 (size=6033)
2024-12-09T17:08:46,414 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/7dcc5252983b4e0d99b8e8b65971a174
2024-12-09T17:08:46,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/7dcc5252983b4e0d99b8e8b65971a174 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/7dcc5252983b4e0d99b8e8b65971a174
2024-12-09T17:08:46,427 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/7dcc5252983b4e0d99b8e8b65971a174, entries=1, sequenceid=18, filesize=5.9 K
2024-12-09T17:08:46,428 INFO  [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 2459d518b0bf9e0f10999c1bed6ae018 in 28ms, sequenceid=18, compaction requested=false
2024-12-09T17:08:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:08:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21
2024-12-09T17:08:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.HMaster(4106): Remote procedure done, pid=21
2024-12-09T17:08:46,432 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20
2024-12-09T17:08:46,432 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 184 msec
2024-12-09T17:08:46,434 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 189 msec
2024-12-09T17:08:46,906 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:47,906 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:48,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:49,909 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:50,910 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:51,912 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:52,913 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:53,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:54,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:08:54,915 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:55,917 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36389 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20
2024-12-09T17:08:56,249 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 20 completed
2024-12-09T17:08:56,254 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C46369%2C1733764074461.1733764136253
2024-12-09T17:08:56,261 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764126233 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764136253
2024-12-09T17:08:56,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43301:43301),(127.0.0.1/127.0.0.1:35731:35731)]
2024-12-09T17:08:56,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764126233 is not closed yet, will try archiving it next time
2024-12-09T17:08:56,261 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461/80c69eb3c456%2C46369%2C1733764074461.1733764116179 to hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/oldWALs/80c69eb3c456%2C46369%2C1733764074461.1733764116179
2024-12-09T17:08:56,261 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-09T17:08:56,261 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-09T17:08:56,261 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58c48638 to 127.0.0.1:53522
2024-12-09T17:08:56,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:08:56,261 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-09T17:08:56,261 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=275959188, stopped=false
2024-12-09T17:08:56,261 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=80c69eb3c456,36389,1733764074329
2024-12-09T17:08:56,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741845_1021 (size=2026)
2024-12-09T17:08:56,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741845_1021 (size=2026)
2024-12-09T17:08:56,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:08:56,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:08:56,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:56,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:56,268 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-09T17:08:56,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:08:56,268 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,46369,1733764074461' *****
2024-12-09T17:08:56,268 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:08:56,268 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:08:56,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:08:56,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:08:56,269 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(3579): Received CLOSE for 2459d518b0bf9e0f10999c1bed6ae018
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(3579): Received CLOSE for 5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,46369,1733764074461
2024-12-09T17:08:56,269 DEBUG [RS:0;80c69eb3c456:46369 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:08:56,269 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2459d518b0bf9e0f10999c1bed6ae018, disabling compactions & flushes
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:08:56,269 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:08:56,269 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:56,269 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018. after waiting 0 ms
2024-12-09T17:08:56,269 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:08:56,269 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:56,270 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2459d518b0bf9e0f10999c1bed6ae018 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-09T17:08:56,270 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-09T17:08:56,270 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1603): Online Regions={2459d518b0bf9e0f10999c1bed6ae018=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018., 5bcbb40859410c4475bd141ae8db5938=hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938., 1588230740=hbase:meta,,1.1588230740}
2024-12-09T17:08:56,270 DEBUG [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 2459d518b0bf9e0f10999c1bed6ae018, 5bcbb40859410c4475bd141ae8db5938
2024-12-09T17:08:56,270 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:08:56,270 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:08:56,270 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:08:56,270 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:08:56,270 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:08:56,270 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=3.05 KB heapSize=5.55 KB
2024-12-09T17:08:56,273 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/362d82fb3a164f76ae352bf9f81e710a is 1080, key is row0001/info:/1733764136250/Put/seqid=0
2024-12-09T17:08:56,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741848_1024 (size=6033)
2024-12-09T17:08:56,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741848_1024 (size=6033)
2024-12-09T17:08:56,278 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/362d82fb3a164f76ae352bf9f81e710a
2024-12-09T17:08:56,284 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/.tmp/info/362d82fb3a164f76ae352bf9f81e710a as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/362d82fb3a164f76ae352bf9f81e710a
2024-12-09T17:08:56,285 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/.tmp/info/944e116e726345a1a0ab927910b8debb is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018./info:regioninfo/1733764076496/Put/seqid=0
2024-12-09T17:08:56,290 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/362d82fb3a164f76ae352bf9f81e710a, entries=1, sequenceid=22, filesize=5.9 K
2024-12-09T17:08:56,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741849_1025 (size=8430)
2024-12-09T17:08:56,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741849_1025 (size=8430)
2024-12-09T17:08:56,291 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 2459d518b0bf9e0f10999c1bed6ae018 in 22ms, sequenceid=22, compaction requested=true
2024-12-09T17:08:56,291 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.79 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/.tmp/info/944e116e726345a1a0ab927910b8debb
2024-12-09T17:08:56,291 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/0ec86a0a6661424badce1b77753ff463, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/75f177dc081e4c61bd735fb9e2cdaa9d, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/acc6e9a1777c4959a356dcbc21187415] to archive
2024-12-09T17:08:56,292 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-09T17:08:56,294 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/0ec86a0a6661424badce1b77753ff463 to hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/0ec86a0a6661424badce1b77753ff463
2024-12-09T17:08:56,295 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/75f177dc081e4c61bd735fb9e2cdaa9d to hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/75f177dc081e4c61bd735fb9e2cdaa9d
2024-12-09T17:08:56,296 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/acc6e9a1777c4959a356dcbc21187415 to hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/info/acc6e9a1777c4959a356dcbc21187415
2024-12-09T17:08:56,300 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/2459d518b0bf9e0f10999c1bed6ae018/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-09T17:08:56,300 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:56,300 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2459d518b0bf9e0f10999c1bed6ae018:

2024-12-09T17:08:56,300 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733764076112.2459d518b0bf9e0f10999c1bed6ae018.
2024-12-09T17:08:56,300 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5bcbb40859410c4475bd141ae8db5938, disabling compactions & flushes
2024-12-09T17:08:56,300 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:56,301 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:56,301 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938. after waiting 0 ms
2024-12-09T17:08:56,301 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:56,304 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/namespace/5bcbb40859410c4475bd141ae8db5938/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-09T17:08:56,304 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:56,305 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5bcbb40859410c4475bd141ae8db5938:

2024-12-09T17:08:56,305 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733764075495.5bcbb40859410c4475bd141ae8db5938.
2024-12-09T17:08:56,309 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/.tmp/table/7c3d35a5470247f29bd14968be8b7725 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733764076503/Put/seqid=0
2024-12-09T17:08:56,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741850_1026 (size=5532)
2024-12-09T17:08:56,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741850_1026 (size=5532)
2024-12-09T17:08:56,313 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=264 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/.tmp/table/7c3d35a5470247f29bd14968be8b7725
2024-12-09T17:08:56,319 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/.tmp/info/944e116e726345a1a0ab927910b8debb as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/info/944e116e726345a1a0ab927910b8debb
2024-12-09T17:08:56,324 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/info/944e116e726345a1a0ab927910b8debb, entries=20, sequenceid=14, filesize=8.2 K
2024-12-09T17:08:56,325 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/.tmp/table/7c3d35a5470247f29bd14968be8b7725 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/table/7c3d35a5470247f29bd14968be8b7725
2024-12-09T17:08:56,331 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/table/7c3d35a5470247f29bd14968be8b7725, entries=4, sequenceid=14, filesize=5.4 K
2024-12-09T17:08:56,332 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~3.05 KB/3122, heapSize ~5.27 KB/5400, currentSize=0 B/0 for 1588230740 in 62ms, sequenceid=14, compaction requested=false
2024-12-09T17:08:56,336 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1
2024-12-09T17:08:56,337 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-09T17:08:56,337 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:08:56,337 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:08:56,337 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-09T17:08:56,470 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,46369,1733764074461; all regions closed.
2024-12-09T17:08:56,471 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461
2024-12-09T17:08:56,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741834_1010 (size=4570)
2024-12-09T17:08:56,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741834_1010 (size=4570)
2024-12-09T17:08:56,481 DEBUG [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/oldWALs
2024-12-09T17:08:56,481 INFO  [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C46369%2C1733764074461.meta:.meta(num 1733764075399)
2024-12-09T17:08:56,481 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/WALs/80c69eb3c456,46369,1733764074461
2024-12-09T17:08:56,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741847_1023 (size=1545)
2024-12-09T17:08:56,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741847_1023 (size=1545)
2024-12-09T17:08:56,487 DEBUG [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(1071): Moved 2 WAL file(s) to /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/oldWALs
2024-12-09T17:08:56,487 INFO  [RS:0;80c69eb3c456:46369 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C46369%2C1733764074461:(num 1733764136253)
2024-12-09T17:08:56,487 DEBUG [RS:0;80c69eb3c456:46369 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:08:56,487 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:08:56,487 INFO  [RS:0;80c69eb3c456:46369 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-09T17:08:56,487 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:08:56,488 INFO  [RS:0;80c69eb3c456:46369 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46369
2024-12-09T17:08:56,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:08:56,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,46369,1733764074461
2024-12-09T17:08:56,493 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,46369,1733764074461]
2024-12-09T17:08:56,493 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,46369,1733764074461; numProcessing=1
2024-12-09T17:08:56,510 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,46369,1733764074461 already deleted, retry=false
2024-12-09T17:08:56,510 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,46369,1733764074461 expired; onlineServers=0
2024-12-09T17:08:56,510 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,36389,1733764074329' *****
2024-12-09T17:08:56,510 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-09T17:08:56,510 DEBUG [M:0;80c69eb3c456:36389 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd25067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:08:56,510 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,36389,1733764074329
2024-12-09T17:08:56,510 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,36389,1733764074329; all regions closed.
2024-12-09T17:08:56,510 DEBUG [M:0;80c69eb3c456:36389 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:08:56,510 DEBUG [M:0;80c69eb3c456:36389 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-09T17:08:56,511 DEBUG [M:0;80c69eb3c456:36389 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-09T17:08:56,511 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-09T17:08:56,511 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764074738 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764074738,5,FailOnTimeoutGroup]
2024-12-09T17:08:56,511 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764074739 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764074739,5,FailOnTimeoutGroup]
2024-12-09T17:08:56,511 INFO  [M:0;80c69eb3c456:36389 {}] hbase.ChoreService(370): Chore service for: master/80c69eb3c456:0 had [] on shutdown
2024-12-09T17:08:56,511 DEBUG [M:0;80c69eb3c456:36389 {}] master.HMaster(1733): Stopping service threads
2024-12-09T17:08:56,511 INFO  [M:0;80c69eb3c456:36389 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-09T17:08:56,511 INFO  [M:0;80c69eb3c456:36389 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-09T17:08:56,512 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-09T17:08:56,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-09T17:08:56,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:56,518 DEBUG [M:0;80c69eb3c456:36389 {}] zookeeper.ZKUtil(347): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-09T17:08:56,518 WARN  [M:0;80c69eb3c456:36389 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-09T17:08:56,518 INFO  [M:0;80c69eb3c456:36389 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-09T17:08:56,518 INFO  [M:0;80c69eb3c456:36389 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-09T17:08:56,518 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:08:56,519 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:56,519 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:56,519 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:08:56,519 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:56,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:08:56,519 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=65.08 KB heapSize=81.70 KB
2024-12-09T17:08:56,538 DEBUG [M:0;80c69eb3c456:36389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3743c7dc6b98437dbdb73fbeb1e25bdc is 82, key is hbase:meta,,1/info:regioninfo/1733764075426/Put/seqid=0
2024-12-09T17:08:56,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741851_1027 (size=5672)
2024-12-09T17:08:56,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741851_1027 (size=5672)
2024-12-09T17:08:56,543 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3743c7dc6b98437dbdb73fbeb1e25bdc
2024-12-09T17:08:56,562 DEBUG [M:0;80c69eb3c456:36389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/352d08d5b44b44798fb5b6e59df38b11 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733764076513/Put/seqid=0
2024-12-09T17:08:56,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741852_1028 (size=8356)
2024-12-09T17:08:56,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741852_1028 (size=8356)
2024-12-09T17:08:56,567 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.47 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/352d08d5b44b44798fb5b6e59df38b11
2024-12-09T17:08:56,571 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 352d08d5b44b44798fb5b6e59df38b11
2024-12-09T17:08:56,584 DEBUG [M:0;80c69eb3c456:36389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b5588b05d4614f4f8d5914269a9da4f7 is 69, key is 80c69eb3c456,46369,1733764074461/rs:state/1733764074817/Put/seqid=0
2024-12-09T17:08:56,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741853_1029 (size=5156)
2024-12-09T17:08:56,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741853_1029 (size=5156)
2024-12-09T17:08:56,589 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b5588b05d4614f4f8d5914269a9da4f7
2024-12-09T17:08:56,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:08:56,602 INFO  [RS:0;80c69eb3c456:46369 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,46369,1733764074461; zookeeper connection closed.
2024-12-09T17:08:56,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46369-0x1000bc367c70001, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:08:56,602 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@302120bf {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@302120bf
2024-12-09T17:08:56,602 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-09T17:08:56,608 DEBUG [M:0;80c69eb3c456:36389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a10f98210fb64ccfa3788fa4938ffedd is 52, key is load_balancer_on/state:d/1733764076106/Put/seqid=0
2024-12-09T17:08:56,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741854_1030 (size=5056)
2024-12-09T17:08:56,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741854_1030 (size=5056)
2024-12-09T17:08:56,613 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a10f98210fb64ccfa3788fa4938ffedd
2024-12-09T17:08:56,618 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3743c7dc6b98437dbdb73fbeb1e25bdc as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3743c7dc6b98437dbdb73fbeb1e25bdc
2024-12-09T17:08:56,623 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3743c7dc6b98437dbdb73fbeb1e25bdc, entries=8, sequenceid=184, filesize=5.5 K
2024-12-09T17:08:56,624 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/352d08d5b44b44798fb5b6e59df38b11 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/352d08d5b44b44798fb5b6e59df38b11
2024-12-09T17:08:56,628 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 352d08d5b44b44798fb5b6e59df38b11
2024-12-09T17:08:56,628 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/352d08d5b44b44798fb5b6e59df38b11, entries=21, sequenceid=184, filesize=8.2 K
2024-12-09T17:08:56,629 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b5588b05d4614f4f8d5914269a9da4f7 as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b5588b05d4614f4f8d5914269a9da4f7
2024-12-09T17:08:56,634 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b5588b05d4614f4f8d5914269a9da4f7, entries=1, sequenceid=184, filesize=5.0 K
2024-12-09T17:08:56,635 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a10f98210fb64ccfa3788fa4938ffedd as hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a10f98210fb64ccfa3788fa4938ffedd
2024-12-09T17:08:56,640 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43331/user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a10f98210fb64ccfa3788fa4938ffedd, entries=1, sequenceid=184, filesize=4.9 K
2024-12-09T17:08:56,642 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(3040): Finished flush of dataSize ~65.08 KB/66637, heapSize ~81.64 KB/83600, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=184, compaction requested=false
2024-12-09T17:08:56,643 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:56,643 DEBUG [M:0;80c69eb3c456:36389 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:08:56,644 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/bcb92887-8020-509f-6e75-0caac092618e/MasterData/WALs/80c69eb3c456,36389,1733764074329
2024-12-09T17:08:56,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43119 is added to blk_1073741830_1006 (size=79158)
2024-12-09T17:08:56,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45869 is added to blk_1073741830_1006 (size=79158)
2024-12-09T17:08:56,646 INFO  [M:0;80c69eb3c456:36389 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-09T17:08:56,646 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:08:56,646 INFO  [M:0;80c69eb3c456:36389 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36389
2024-12-09T17:08:56,651 DEBUG [M:0;80c69eb3c456:36389 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/80c69eb3c456,36389,1733764074329 already deleted, retry=false
2024-12-09T17:08:56,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:08:56,760 INFO  [M:0;80c69eb3c456:36389 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,36389,1733764074329; zookeeper connection closed.
2024-12-09T17:08:56,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36389-0x1000bc367c70000, quorum=127.0.0.1:53522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:08:56,767 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@61ea8370{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:08:56,768 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2959bd10{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:08:56,768 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:08:56,768 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@541bc67d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:08:56,769 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1611ccec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir/,STOPPED}
2024-12-09T17:08:56,770 WARN  [BP-850976073-172.17.0.2-1733764072624 heartbeating to localhost/127.0.0.1:43331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:08:56,770 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:08:56,770 WARN  [BP-850976073-172.17.0.2-1733764072624 heartbeating to localhost/127.0.0.1:43331 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-850976073-172.17.0.2-1733764072624 (Datanode Uuid 04db581a-212a-453d-aa85-af8de0141933) service to localhost/127.0.0.1:43331
2024-12-09T17:08:56,770 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:08:56,771 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data3/current/BP-850976073-172.17.0.2-1733764072624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:08:56,771 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data4/current/BP-850976073-172.17.0.2-1733764072624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:08:56,771 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:08:56,773 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b4b904e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:08:56,774 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@50f9eea5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:08:56,774 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:08:56,774 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36119ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:08:56,774 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7145ab8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir/,STOPPED}
2024-12-09T17:08:56,775 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:08:56,775 WARN  [BP-850976073-172.17.0.2-1733764072624 heartbeating to localhost/127.0.0.1:43331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:08:56,775 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:08:56,775 WARN  [BP-850976073-172.17.0.2-1733764072624 heartbeating to localhost/127.0.0.1:43331 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-850976073-172.17.0.2-1733764072624 (Datanode Uuid 561690cb-6d19-4005-b2af-ac2472793b59) service to localhost/127.0.0.1:43331
2024-12-09T17:08:56,776 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data1/current/BP-850976073-172.17.0.2-1733764072624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:08:56,777 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/cluster_cde7be64-0215-a6c7-821c-1f65078c24da/dfs/data/data2/current/BP-850976073-172.17.0.2-1733764072624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:08:56,777 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:08:56,785 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58ac6791{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:08:56,786 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c7be892{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:08:56,786 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:08:56,786 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@262a1159{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:08:56,786 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22a9191b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir/,STOPPED}
2024-12-09T17:08:56,791 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-09T17:08:56,807 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-09T17:08:56,812 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=109 (was 101) - Thread LEAK? -, OpenFileDescriptor=464 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=25 (was 29), ProcessCount=11 (was 11), AvailableMemoryMB=4860 (was 4951)
2024-12-09T17:08:56,817 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=110, OpenFileDescriptor=464, MaxFileDescriptor=1048576, SystemLoadAverage=25, ProcessCount=11, AvailableMemoryMB=4859
2024-12-09T17:08:56,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-09T17:08:56,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.log.dir so I do NOT create it in target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87
2024-12-09T17:08:56,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9e3fd0cb-6648-fcdd-0bf8-51de87a4d336/hadoop.tmp.dir so I do NOT create it in target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87
2024-12-09T17:08:56,817 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6, deleteOnExit=true
2024-12-09T17:08:56,817 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/test.cache.data in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.tmp.dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-09T17:08:56,818 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:08:56,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-09T17:08:56,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/nfs.dump.dir in system properties and HBase conf
2024-12-09T17:08:56,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/java.io.tmpdir in system properties and HBase conf
2024-12-09T17:08:56,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:08:56,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-09T17:08:56,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-09T17:08:56,829 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:08:56,846 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:08:56,918 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:57,126 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:08:57,129 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:08:57,130 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:08:57,130 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:08:57,130 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:08:57,130 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:08:57,131 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e41ff6b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:08:57,131 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73d21e1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:08:57,219 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c76d837{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/java.io.tmpdir/jetty-localhost-45707-hadoop-hdfs-3_4_1-tests_jar-_-any-10844796248737875920/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:08:57,220 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a029663{HTTP/1.1, (http/1.1)}{localhost:45707}
2024-12-09T17:08:57,220 INFO  [Time-limited test {}] server.Server(415): Started @291639ms
2024-12-09T17:08:57,229 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:08:57,392 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:08:57,395 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:08:57,395 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:08:57,395 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:08:57,395 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:08:57,396 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e8f61f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:08:57,396 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7aa85543{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:08:57,486 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77ee6c4c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/java.io.tmpdir/jetty-localhost-35533-hadoop-hdfs-3_4_1-tests_jar-_-any-17138135909013728220/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:08:57,486 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b54bf6a{HTTP/1.1, (http/1.1)}{localhost:35533}
2024-12-09T17:08:57,486 INFO  [Time-limited test {}] server.Server(415): Started @291906ms
2024-12-09T17:08:57,487 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:08:57,514 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:08:57,517 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:08:57,517 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:08:57,517 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:08:57,518 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:08:57,518 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54b978f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:08:57,518 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a4f2efe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:08:57,606 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@52c967b0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/java.io.tmpdir/jetty-localhost-36397-hadoop-hdfs-3_4_1-tests_jar-_-any-1774130957800874526/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:08:57,606 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3dfee23a{HTTP/1.1, (http/1.1)}{localhost:36397}
2024-12-09T17:08:57,606 INFO  [Time-limited test {}] server.Server(415): Started @292026ms
2024-12-09T17:08:57,607 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:08:57,919 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:58,327 WARN  [Thread-1737 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data1/current/BP-1569817708-172.17.0.2-1733764136840/current, will proceed with Du for space computation calculation, 
2024-12-09T17:08:58,327 WARN  [Thread-1738 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data2/current/BP-1569817708-172.17.0.2-1733764136840/current, will proceed with Du for space computation calculation, 
2024-12-09T17:08:58,351 WARN  [Thread-1701 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:08:58,354 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x89439ef19d35d27a with lease ID 0xd2f4d3f4ab95dd3e: Processing first storage report for DS-ae451fe4-5918-4496-948e-797879a3f983 from datanode DatanodeRegistration(127.0.0.1:45139, datanodeUuid=4251a0ef-e81d-4f68-9d1d-43dcaa50ffd1, infoPort=41693, infoSecurePort=0, ipcPort=45561, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840)
2024-12-09T17:08:58,354 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x89439ef19d35d27a with lease ID 0xd2f4d3f4ab95dd3e: from storage DS-ae451fe4-5918-4496-948e-797879a3f983 node DatanodeRegistration(127.0.0.1:45139, datanodeUuid=4251a0ef-e81d-4f68-9d1d-43dcaa50ffd1, infoPort=41693, infoSecurePort=0, ipcPort=45561, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:08:58,354 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x89439ef19d35d27a with lease ID 0xd2f4d3f4ab95dd3e: Processing first storage report for DS-ad8015ba-da03-4338-91a9-e6d31c42db09 from datanode DatanodeRegistration(127.0.0.1:45139, datanodeUuid=4251a0ef-e81d-4f68-9d1d-43dcaa50ffd1, infoPort=41693, infoSecurePort=0, ipcPort=45561, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840)
2024-12-09T17:08:58,354 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x89439ef19d35d27a with lease ID 0xd2f4d3f4ab95dd3e: from storage DS-ad8015ba-da03-4338-91a9-e6d31c42db09 node DatanodeRegistration(127.0.0.1:45139, datanodeUuid=4251a0ef-e81d-4f68-9d1d-43dcaa50ffd1, infoPort=41693, infoSecurePort=0, ipcPort=45561, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:08:58,428 WARN  [Thread-1748 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data3/current/BP-1569817708-172.17.0.2-1733764136840/current, will proceed with Du for space computation calculation, 
2024-12-09T17:08:58,428 WARN  [Thread-1749 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data4/current/BP-1569817708-172.17.0.2-1733764136840/current, will proceed with Du for space computation calculation, 
2024-12-09T17:08:58,446 WARN  [Thread-1724 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:08:58,448 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x607b22ea71b6060d with lease ID 0xd2f4d3f4ab95dd3f: Processing first storage report for DS-552069c6-e68b-48d4-be47-95802b889fab from datanode DatanodeRegistration(127.0.0.1:44491, datanodeUuid=48439287-f2c8-45d9-8aca-a98463b0e34f, infoPort=36441, infoSecurePort=0, ipcPort=42491, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840)
2024-12-09T17:08:58,448 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x607b22ea71b6060d with lease ID 0xd2f4d3f4ab95dd3f: from storage DS-552069c6-e68b-48d4-be47-95802b889fab node DatanodeRegistration(127.0.0.1:44491, datanodeUuid=48439287-f2c8-45d9-8aca-a98463b0e34f, infoPort=36441, infoSecurePort=0, ipcPort=42491, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:08:58,448 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x607b22ea71b6060d with lease ID 0xd2f4d3f4ab95dd3f: Processing first storage report for DS-becb8592-33a7-45e1-b7ef-197522e72680 from datanode DatanodeRegistration(127.0.0.1:44491, datanodeUuid=48439287-f2c8-45d9-8aca-a98463b0e34f, infoPort=36441, infoSecurePort=0, ipcPort=42491, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840)
2024-12-09T17:08:58,448 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x607b22ea71b6060d with lease ID 0xd2f4d3f4ab95dd3f: from storage DS-becb8592-33a7-45e1-b7ef-197522e72680 node DatanodeRegistration(127.0.0.1:44491, datanodeUuid=48439287-f2c8-45d9-8aca-a98463b0e34f, infoPort=36441, infoSecurePort=0, ipcPort=42491, storageInfo=lv=-57;cid=testClusterID;nsid=1664945951;c=1733764136840), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:08:58,532 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87
2024-12-09T17:08:58,535 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/zookeeper_0, clientPort=49603, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-09T17:08:58,536 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49603
2024-12-09T17:08:58,536 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,538 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:08:58,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:08:58,547 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b with version=8
2024-12-09T17:08:58,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase-staging
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:08:58,549 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:08:58,550 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46085
2024-12-09T17:08:58,550 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,552 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,554 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:46085 connecting to ZooKeeper ensemble=127.0.0.1:49603
2024-12-09T17:08:58,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:460850x0, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:08:58,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46085-0x1000bc462a70000 connected
2024-12-09T17:08:58,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:08:58,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:08:58,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:08:58,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46085
2024-12-09T17:08:58,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46085
2024-12-09T17:08:58,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46085
2024-12-09T17:08:58,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46085
2024-12-09T17:08:58,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46085
2024-12-09T17:08:58,673 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b, hbase.cluster.distributed=false
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:08:58,692 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:08:58,693 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:08:58,693 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43437
2024-12-09T17:08:58,693 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:08:58,694 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:08:58,694 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,696 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,697 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:43437 connecting to ZooKeeper ensemble=127.0.0.1:49603
2024-12-09T17:08:58,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434370x0, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:08:58,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43437-0x1000bc462a70001 connected
2024-12-09T17:08:58,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:08:58,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:08:58,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:08:58,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43437
2024-12-09T17:08:58,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43437
2024-12-09T17:08:58,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43437
2024-12-09T17:08:58,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43437
2024-12-09T17:08:58,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43437
2024-12-09T17:08:58,705 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:08:58,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:08:58,710 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,716 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80c69eb3c456:46085
2024-12-09T17:08:58,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:08:58,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:08:58,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,718 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:08:58,719 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80c69eb3c456,46085,1733764138548 from backup master directory
2024-12-09T17:08:58,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:08:58,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:08:58,726 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:08:58,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:08:58,726 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:08:58,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:08:58,738 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/hbase.id with ID: a0cc21f9-2394-4501-95db-0e3f1a4a87a4
2024-12-09T17:08:58,751 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:08:58,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:08:58,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:08:58,768 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:08:58,769 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-09T17:08:58,769 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:08:58,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:08:58,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:08:58,779 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store
2024-12-09T17:08:58,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:08:58,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:08:58,786 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:08:58,786 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:08:58,786 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:58,787 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:58,787 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:08:58,787 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:58,787 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:08:58,787 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:08:58,787 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/.initializing
2024-12-09T17:08:58,788 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/WALs/80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,790 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C46085%2C1733764138548, suffix=, logDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/WALs/80c69eb3c456,46085,1733764138548, archiveDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/oldWALs, maxLogs=10
2024-12-09T17:08:58,791 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C46085%2C1733764138548.1733764138790
2024-12-09T17:08:58,797 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/WALs/80c69eb3c456,46085,1733764138548/80c69eb3c456%2C46085%2C1733764138548.1733764138790
2024-12-09T17:08:58,797 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36441:36441),(127.0.0.1/127.0.0.1:41693:41693)]
2024-12-09T17:08:58,797 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:08:58,797 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:08:58,797 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:08:58,797 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:08:58,799 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:08:58,801 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-09T17:08:58,801 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,802 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:58,802 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:08:58,803 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-09T17:08:58,803 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,804 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:08:58,804 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:08:58,805 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-09T17:08:58,805 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,805 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:08:58,805 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:08:58,806 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-09T17:08:58,806 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,807 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:08:58,807 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:08:58,808 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:08:58,809 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-09T17:08:58,810 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:08:58,811 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:08:58,812 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782677, jitterRate=-0.004775121808052063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-09T17:08:58,812 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:08:58,812 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-09T17:08:58,815 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41df9d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:08:58,816 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-09T17:08:58,816 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-09T17:08:58,816 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-09T17:08:58,816 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-09T17:08:58,817 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-09T17:08:58,817 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-09T17:08:58,817 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-09T17:08:58,819 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-09T17:08:58,819 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-09T17:08:58,826 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-09T17:08:58,827 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-09T17:08:58,828 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-09T17:08:58,835 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-09T17:08:58,835 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-09T17:08:58,836 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-09T17:08:58,843 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-09T17:08:58,844 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-09T17:08:58,851 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-09T17:08:58,854 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-09T17:08:58,860 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-09T17:08:58,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:08:58,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:08:58,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,869 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=80c69eb3c456,46085,1733764138548, sessionid=0x1000bc462a70000, setting cluster-up flag (Was=false)
2024-12-09T17:08:58,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,910 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-09T17:08:58,914 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:08:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:58,960 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-09T17:08:58,961 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,46085,1733764138548
2024-12-09T17:08:58,963 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-09T17:08:58,964 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-09T17:08:58,964 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-09T17:08:58,964 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80c69eb3c456,46085,1733764138548
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-09T17:08:58,964 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:08:58,964 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:08:58,964 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:08:58,964 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:08:58,965 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80c69eb3c456:0, corePoolSize=10, maxPoolSize=10
2024-12-09T17:08:58,965 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:58,965 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:08:58,965 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733764168966
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-09T17:08:58,966 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-09T17:08:58,966 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-09T17:08:58,966 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:58,967 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-09T17:08:58,967 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-09T17:08:58,967 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-09T17:08:58,967 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-09T17:08:58,967 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-09T17:08:58,968 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764138967,5,FailOnTimeoutGroup]
2024-12-09T17:08:58,968 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,968 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764138968,5,FailOnTimeoutGroup]
2024-12-09T17:08:58,968 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:58,968 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:08:58,968 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-09T17:08:58,968 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:58,968 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:58,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:08:58,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:08:58,974 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-09T17:08:58,974 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b
2024-12-09T17:08:58,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:08:58,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:08:58,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:08:58,982 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:08:58,983 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:08:58,984 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,984 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:58,984 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:08:58,985 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:08:58,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,986 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:58,986 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:08:58,987 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:08:58,987 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:58,988 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:58,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740
2024-12-09T17:08:58,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740
2024-12-09T17:08:58,990 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:08:58,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:08:58,994 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:08:58,995 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=883238, jitterRate=0.12309616804122925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:08:58,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:08:58,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:08:58,995 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:08:58,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:08:58,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:08:58,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:08:58,995 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:08:58,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:08:58,997 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:08:58,997 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-09T17:08:58,997 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-09T17:08:58,998 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-09T17:08:58,999 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-09T17:08:59,014 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80c69eb3c456:43437
2024-12-09T17:08:59,015 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1008): ClusterId : a0cc21f9-2394-4501-95db-0e3f1a4a87a4
2024-12-09T17:08:59,016 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:08:59,027 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:08:59,027 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:08:59,035 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:08:59,035 DEBUG [RS:0;80c69eb3c456:43437 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e4395e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:08:59,036 DEBUG [RS:0;80c69eb3c456:43437 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b14fbe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:08:59,036 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:08:59,036 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:08:59,036 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:08:59,036 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,46085,1733764138548 with isa=80c69eb3c456/172.17.0.2:43437, startcode=1733764138692
2024-12-09T17:08:59,036 DEBUG [RS:0;80c69eb3c456:43437 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:08:59,039 INFO  [RS-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35139, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:08:59,039 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46085 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,039 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46085 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,041 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b
2024-12-09T17:08:59,041 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35833
2024-12-09T17:08:59,041 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:08:59,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:08:59,051 DEBUG [RS:0;80c69eb3c456:43437 {}] zookeeper.ZKUtil(111): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,051 WARN  [RS:0;80c69eb3c456:43437 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:08:59,051 INFO  [RS:0;80c69eb3c456:43437 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:08:59,051 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,051 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,43437,1733764138692]
2024-12-09T17:08:59,054 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:08:59,054 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:08:59,075 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:08:59,075 INFO  [RS:0;80c69eb3c456:43437 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:08:59,076 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,076 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:08:59,076 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:08:59,077 DEBUG [RS:0;80c69eb3c456:43437 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:08:59,077 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,077 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,077 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,077 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,078 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,43437,1733764138692-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:08:59,090 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:08:59,090 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,43437,1733764138692-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,101 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.Replication(204): 80c69eb3c456,43437,1733764138692 started
2024-12-09T17:08:59,101 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,43437,1733764138692, RpcServer on 80c69eb3c456/172.17.0.2:43437, sessionid=0x1000bc462a70001
2024-12-09T17:08:59,101 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:08:59,101 DEBUG [RS:0;80c69eb3c456:43437 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,101 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,43437,1733764138692'
2024-12-09T17:08:59,101 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:08:59,102 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:08:59,102 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:08:59,102 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:08:59,102 DEBUG [RS:0;80c69eb3c456:43437 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,102 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,43437,1733764138692'
2024-12-09T17:08:59,102 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:08:59,103 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:08:59,103 DEBUG [RS:0;80c69eb3c456:43437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:08:59,103 INFO  [RS:0;80c69eb3c456:43437 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:08:59,103 INFO  [RS:0;80c69eb3c456:43437 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:08:59,149 WARN  [80c69eb3c456:46085 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-09T17:08:59,206 INFO  [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C43437%2C1733764138692, suffix=, logDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692, archiveDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/oldWALs, maxLogs=32
2024-12-09T17:08:59,207 INFO  [RS:0;80c69eb3c456:43437 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C43437%2C1733764138692.1733764139207
2024-12-09T17:08:59,216 INFO  [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764139207
2024-12-09T17:08:59,217 DEBUG [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41693:41693),(127.0.0.1/127.0.0.1:36441:36441)]
2024-12-09T17:08:59,400 DEBUG [80c69eb3c456:46085 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-09T17:08:59,400 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,403 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,43437,1733764138692, state=OPENING
2024-12-09T17:08:59,443 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-09T17:08:59,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:59,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:08:59,454 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=80c69eb3c456,43437,1733764138692}]
2024-12-09T17:08:59,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:08:59,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:08:59,610 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,610 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-09T17:08:59,615 INFO  [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37858, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-09T17:08:59,620 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-09T17:08:59,620 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:08:59,623 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C43437%2C1733764138692.meta, suffix=.meta, logDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692, archiveDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/oldWALs, maxLogs=32
2024-12-09T17:08:59,624 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C43437%2C1733764138692.meta.1733764139623.meta
2024-12-09T17:08:59,629 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.meta.1733764139623.meta
2024-12-09T17:08:59,629 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36441:36441),(127.0.0.1/127.0.0.1:41693:41693)]
2024-12-09T17:08:59,629 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:08:59,629 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-09T17:08:59,629 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-09T17:08:59,629 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-09T17:08:59,629 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-09T17:08:59,630 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:08:59,630 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-09T17:08:59,630 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-09T17:08:59,631 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:08:59,632 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:08:59,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:59,633 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:59,633 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:08:59,633 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:08:59,633 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:59,634 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:59,634 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:08:59,635 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:08:59,635 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:59,635 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:08:59,636 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740
2024-12-09T17:08:59,637 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740
2024-12-09T17:08:59,638 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:08:59,639 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:08:59,640 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866544, jitterRate=0.10186871886253357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:08:59,640 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:08:59,641 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733764139609
2024-12-09T17:08:59,642 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-09T17:08:59,642 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-09T17:08:59,643 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,643 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,43437,1733764138692, state=OPEN
2024-12-09T17:08:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:08:59,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:08:59,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:08:59,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:08:59,667 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-09T17:08:59,667 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=80c69eb3c456,43437,1733764138692 in 212 msec
2024-12-09T17:08:59,669 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-09T17:08:59,669 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 670 msec
2024-12-09T17:08:59,670 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 707 msec
2024-12-09T17:08:59,670 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733764139670, completionTime=-1
2024-12-09T17:08:59,670 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-09T17:08:59,670 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-09T17:08:59,671 DEBUG [hconnection-0x1a4e340-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:08:59,672 INFO  [RS-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37868, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:08:59,673 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-09T17:08:59,673 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733764199673
2024-12-09T17:08:59,673 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733764259673
2024-12-09T17:08:59,673 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec
2024-12-09T17:08:59,693 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46085,1733764138548-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,693 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46085,1733764138548-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,693 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46085,1733764138548-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,694 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80c69eb3c456:46085, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,694 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-09T17:08:59,694 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-09T17:08:59,694 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:08:59,695 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-09T17:08:59,695 DEBUG [master/80c69eb3c456:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-09T17:08:59,696 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:08:59,696 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:08:59,697 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:08:59,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:08:59,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:08:59,704 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => deb7453c97cf6e1b1675a89c6a7593bf, NAME => 'hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b
2024-12-09T17:08:59,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:08:59,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:08:59,712 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:08:59,712 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing deb7453c97cf6e1b1675a89c6a7593bf, disabling compactions & flushes
2024-12-09T17:08:59,712 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:08:59,712 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:08:59,712 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf. after waiting 0 ms
2024-12-09T17:08:59,712 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:08:59,712 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:08:59,712 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for deb7453c97cf6e1b1675a89c6a7593bf:

2024-12-09T17:08:59,713 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:08:59,713 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733764139713"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764139713"}]},"ts":"1733764139713"}
2024-12-09T17:08:59,715 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:08:59,716 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:08:59,716 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764139716"}]},"ts":"1733764139716"}
2024-12-09T17:08:59,718 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-09T17:08:59,735 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=deb7453c97cf6e1b1675a89c6a7593bf, ASSIGN}]
2024-12-09T17:08:59,736 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=deb7453c97cf6e1b1675a89c6a7593bf, ASSIGN
2024-12-09T17:08:59,737 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=deb7453c97cf6e1b1675a89c6a7593bf, ASSIGN; state=OFFLINE, location=80c69eb3c456,43437,1733764138692; forceNewPlan=false, retain=false
2024-12-09T17:08:59,888 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=deb7453c97cf6e1b1675a89c6a7593bf, regionState=OPENING, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:08:59,892 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure deb7453c97cf6e1b1675a89c6a7593bf, server=80c69eb3c456,43437,1733764138692}]
2024-12-09T17:08:59,921 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:00,048 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,43437,1733764138692
2024-12-09T17:09:00,058 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:09:00,058 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => deb7453c97cf6e1b1675a89c6a7593bf, NAME => 'hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:09:00,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:09:00,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:09:00,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:09:00,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:09:00,061 INFO  [StoreOpener-deb7453c97cf6e1b1675a89c6a7593bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region deb7453c97cf6e1b1675a89c6a7593bf 
2024-12-09T17:09:00,063 INFO  [StoreOpener-deb7453c97cf6e1b1675a89c6a7593bf-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region deb7453c97cf6e1b1675a89c6a7593bf columnFamilyName info
2024-12-09T17:09:00,064 DEBUG [StoreOpener-deb7453c97cf6e1b1675a89c6a7593bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:00,064 INFO  [StoreOpener-deb7453c97cf6e1b1675a89c6a7593bf-1 {}] regionserver.HStore(327): Store=deb7453c97cf6e1b1675a89c6a7593bf/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:09:00,065 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:09:00,066 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:09:00,069 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:09:00,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:09:00,073 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened deb7453c97cf6e1b1675a89c6a7593bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756899, jitterRate=-0.03755423426628113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:09:00,073 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for deb7453c97cf6e1b1675a89c6a7593bf:

2024-12-09T17:09:00,073 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf., pid=6, masterSystemTime=1733764140048
2024-12-09T17:09:00,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:09:00,075 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:09:00,075 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=deb7453c97cf6e1b1675a89c6a7593bf, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:00,079 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-09T17:09:00,079 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure deb7453c97cf6e1b1675a89c6a7593bf, server=80c69eb3c456,43437,1733764138692 in 185 msec
2024-12-09T17:09:00,081 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-09T17:09:00,081 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=deb7453c97cf6e1b1675a89c6a7593bf, ASSIGN in 344 msec
2024-12-09T17:09:00,081 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:09:00,081 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764140081"}]},"ts":"1733764140081"}
2024-12-09T17:09:00,083 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-09T17:09:00,119 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-09T17:09:00,120 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:09:00,124 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 426 msec
2024-12-09T17:09:00,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:09:00,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:09:00,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:09:00,131 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-09T17:09:00,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:09:00,154 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 22 msec
2024-12-09T17:09:00,163 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-09T17:09:00,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:09:00,189 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 24 msec
2024-12-09T17:09:00,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-09T17:09:00,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-09T17:09:00,226 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.500sec
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46085,1733764138548-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:09:00,227 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46085,1733764138548-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-09T17:09:00,230 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-09T17:09:00,230 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-09T17:09:00,230 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,46085,1733764138548-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:09:00,308 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b07f769 to 127.0.0.1:49603 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e300607
2024-12-09T17:09:00,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2fb7bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:09:00,323 DEBUG [hconnection-0x1d39002b-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:09:00,325 INFO  [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60786, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:09:00,328 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=80c69eb3c456,46085,1733764138548
2024-12-09T17:09:00,328 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:09:00,332 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-09T17:09:00,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-09T17:09:00,335 INFO  [RS-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51612, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-09T17:09:00,337 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-09T17:09:00,337 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-09T17:09:00,337 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:09:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-09T17:09:00,339 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:09:00,339 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:00,340 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 9
2024-12-09T17:09:00,340 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:09:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:09:00,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741837_1013 (size=381)
2024-12-09T17:09:00,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741837_1013 (size=381)
2024-12-09T17:09:00,351 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9b1c820bb7effbbb64e39968a9dba25e, NAME => 'TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b
2024-12-09T17:09:00,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741838_1014 (size=64)
2024-12-09T17:09:00,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741838_1014 (size=64)
2024-12-09T17:09:00,358 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:09:00,358 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 9b1c820bb7effbbb64e39968a9dba25e, disabling compactions & flushes
2024-12-09T17:09:00,359 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,359 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,359 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e. after waiting 0 ms
2024-12-09T17:09:00,359 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,359 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,359 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:00,359 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:09:00,360 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733764140359"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764140359"}]},"ts":"1733764140359"}
2024-12-09T17:09:00,361 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:09:00,361 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:09:00,362 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764140362"}]},"ts":"1733764140362"}
2024-12-09T17:09:00,363 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-09T17:09:00,376 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, ASSIGN}]
2024-12-09T17:09:00,378 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, ASSIGN
2024-12-09T17:09:00,378 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, ASSIGN; state=OFFLINE, location=80c69eb3c456,43437,1733764138692; forceNewPlan=false, retain=false
2024-12-09T17:09:00,529 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=9b1c820bb7effbbb64e39968a9dba25e, regionState=OPENING, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:00,533 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692}]
2024-12-09T17:09:00,689 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,43437,1733764138692
2024-12-09T17:09:00,698 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,698 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 9b1c820bb7effbbb64e39968a9dba25e, NAME => 'TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:09:00,699 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:00,699 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:09:00,699 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:00,699 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:00,702 INFO  [StoreOpener-9b1c820bb7effbbb64e39968a9dba25e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9b1c820bb7effbbb64e39968a9dba25e 
2024-12-09T17:09:00,704 INFO  [StoreOpener-9b1c820bb7effbbb64e39968a9dba25e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b1c820bb7effbbb64e39968a9dba25e columnFamilyName info
2024-12-09T17:09:00,704 DEBUG [StoreOpener-9b1c820bb7effbbb64e39968a9dba25e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:00,705 INFO  [StoreOpener-9b1c820bb7effbbb64e39968a9dba25e-1 {}] regionserver.HStore(327): Store=9b1c820bb7effbbb64e39968a9dba25e/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:09:00,706 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:00,706 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:00,708 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:00,711 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:09:00,711 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 9b1c820bb7effbbb64e39968a9dba25e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=714968, jitterRate=-0.09087128937244415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:09:00,712 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:00,713 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e., pid=11, masterSystemTime=1733764140689
2024-12-09T17:09:00,715 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,715 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:00,715 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=9b1c820bb7effbbb64e39968a9dba25e, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:00,718 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-09T17:09:00,718 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692 in 184 msec
2024-12-09T17:09:00,720 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-09T17:09:00,720 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, ASSIGN in 342 msec
2024-12-09T17:09:00,720 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:09:00,720 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764140720"}]},"ts":"1733764140720"}
2024-12-09T17:09:00,722 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-09T17:09:00,727 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:09:00,728 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRolling in 390 msec
2024-12-09T17:09:00,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:01,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,302 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,302 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,303 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,303 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,324 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,324 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,324 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,326 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,830 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:09:01,831 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,833 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,834 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,851 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,851 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,851 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,856 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:01,860 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-09T17:09:01,860 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-09T17:09:01,860 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-09T17:09:01,923 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:02,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:03,925 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:04,926 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:05,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-09T17:09:05,056 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-09T17:09:05,057 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-09T17:09:05,928 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:06,929 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:07,365 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:09:07,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,386 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:07,930 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:08,931 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:09,932 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46085 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-09T17:09:10,344 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling, procId: 9 completed
2024-12-09T17:09:10,350 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-09T17:09:10,351 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:10,363 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:09:10,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/7c1882986707431a9b27b5311bd4bf5c is 1080, key is row0001/info:/1733764150355/Put/seqid=0
2024-12-09T17:09:10,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741839_1015 (size=12509)
2024-12-09T17:09:10,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741839_1015 (size=12509)
2024-12-09T17:09:10,385 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/7c1882986707431a9b27b5311bd4bf5c
2024-12-09T17:09:10,389 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-09T17:09:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60786 deadline: 1733764160388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/7c1882986707431a9b27b5311bd4bf5c as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/7c1882986707431a9b27b5311bd4bf5c
2024-12-09T17:09:10,398 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/7c1882986707431a9b27b5311bd4bf5c, entries=7, sequenceid=11, filesize=12.2 K
2024-12-09T17:09:10,399 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 9b1c820bb7effbbb64e39968a9dba25e in 36ms, sequenceid=11, compaction requested=false
2024-12-09T17:09:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:10,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:11,792 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=1, created chunk count=12, reused chunk count=39, reuseRatio=76.47%
2024-12-09T17:09:11,792 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-09T17:09:11,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:12,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:13,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:14,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:15,939 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:16,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:17,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:18,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,375 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,393 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,393 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,393 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,399 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,906 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:09:18,908 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,938 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,938 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,938 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,940 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:18,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:19,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:20,481 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-09T17:09:20,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/956cadc4b48b4766acdf99f99477bdf9 is 1080, key is row0008/info:/1733764150363/Put/seqid=0
2024-12-09T17:09:20,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741840_1016 (size=29761)
2024-12-09T17:09:20,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741840_1016 (size=29761)
2024-12-09T17:09:20,495 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/956cadc4b48b4766acdf99f99477bdf9
2024-12-09T17:09:20,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/956cadc4b48b4766acdf99f99477bdf9 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9
2024-12-09T17:09:20,507 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9, entries=23, sequenceid=37, filesize=29.1 K
2024-12-09T17:09:20,508 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 9b1c820bb7effbbb64e39968a9dba25e in 27ms, sequenceid=37, compaction requested=false
2024-12-09T17:09:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=41.3 K, sizeToCheck=16.0 K
2024-12-09T17:09:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:09:20,508 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9 because midkey is the same as first or last row
2024-12-09T17:09:20,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:21,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:22,501 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:09:22,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/12c6f2f7dc20416cbdb13ae82c2adb79 is 1080, key is row0031/info:/1733764160483/Put/seqid=0
2024-12-09T17:09:22,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741841_1017 (size=12509)
2024-12-09T17:09:22,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741841_1017 (size=12509)
2024-12-09T17:09:22,530 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-09T17:09:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60786 deadline: 1733764172530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:22,916 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/12c6f2f7dc20416cbdb13ae82c2adb79
2024-12-09T17:09:22,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/12c6f2f7dc20416cbdb13ae82c2adb79 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/12c6f2f7dc20416cbdb13ae82c2adb79
2024-12-09T17:09:22,935 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/12c6f2f7dc20416cbdb13ae82c2adb79, entries=7, sequenceid=47, filesize=12.2 K
2024-12-09T17:09:22,937 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 9b1c820bb7effbbb64e39968a9dba25e in 435ms, sequenceid=47, compaction requested=true
2024-12-09T17:09:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=53.5 K, sizeToCheck=16.0 K
2024-12-09T17:09:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:09:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9 because midkey is the same as first or last row
2024-12-09T17:09:22,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b1c820bb7effbbb64e39968a9dba25e:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:09:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:22,938 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:09:22,939 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:09:22,939 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): 9b1c820bb7effbbb64e39968a9dba25e/info is initiating minor compaction (all files)
2024-12-09T17:09:22,939 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b1c820bb7effbbb64e39968a9dba25e/info in TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:22,939 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/7c1882986707431a9b27b5311bd4bf5c, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/12c6f2f7dc20416cbdb13ae82c2adb79] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp, totalSize=53.5 K
2024-12-09T17:09:22,940 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7c1882986707431a9b27b5311bd4bf5c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733764150355
2024-12-09T17:09:22,940 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 956cadc4b48b4766acdf99f99477bdf9, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733764150363
2024-12-09T17:09:22,940 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 12c6f2f7dc20416cbdb13ae82c2adb79, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733764160483
2024-12-09T17:09:22,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:22,956 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b1c820bb7effbbb64e39968a9dba25e#info#compaction#44 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:09:22,957 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/02b61623b0db4d5a9a51060795793cd8 is 1080, key is row0001/info:/1733764150355/Put/seqid=0
2024-12-09T17:09:22,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741842_1018 (size=44978)
2024-12-09T17:09:22,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741842_1018 (size=44978)
2024-12-09T17:09:22,969 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/02b61623b0db4d5a9a51060795793cd8 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8
2024-12-09T17:09:22,976 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b1c820bb7effbbb64e39968a9dba25e/info of 9b1c820bb7effbbb64e39968a9dba25e into 02b61623b0db4d5a9a51060795793cd8(size=43.9 K), total size for store is 43.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:09:22,976 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:22,976 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e., storeName=9b1c820bb7effbbb64e39968a9dba25e/info, priority=13, startTime=1733764162937; duration=0sec
2024-12-09T17:09:22,976 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=43.9 K, sizeToCheck=16.0 K
2024-12-09T17:09:22,976 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:09:22,976 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8 because midkey is the same as first or last row
2024-12-09T17:09:22,976 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:22,976 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b1c820bb7effbbb64e39968a9dba25e:info
2024-12-09T17:09:23,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:24,945 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:25,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:26,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:27,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:28,532 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:09:28,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:29,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:30,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:31,036 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:09:31,040 INFO  [RS-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39172, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:09:31,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:32,623 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-09T17:09:32,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/8d682c258b514256b002f37d58973893 is 1080, key is row0038/info:/1733764162502/Put/seqid=0
2024-12-09T17:09:32,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741843_1019 (size=29761)
2024-12-09T17:09:32,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741843_1019 (size=29761)
2024-12-09T17:09:32,636 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/8d682c258b514256b002f37d58973893
2024-12-09T17:09:32,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/8d682c258b514256b002f37d58973893 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/8d682c258b514256b002f37d58973893
2024-12-09T17:09:32,649 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/8d682c258b514256b002f37d58973893, entries=23, sequenceid=74, filesize=29.1 K
2024-12-09T17:09:32,650 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for 9b1c820bb7effbbb64e39968a9dba25e in 27ms, sequenceid=74, compaction requested=false
2024-12-09T17:09:32,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:32,651 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=73.0 K, sizeToCheck=16.0 K
2024-12-09T17:09:32,651 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:09:32,651 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8 because midkey is the same as first or last row
2024-12-09T17:09:32,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:33,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:34,639 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:09:34,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/56a145a4a3fd4e429a2667299d1c88a7 is 1080, key is row0061/info:/1733764172624/Put/seqid=0
2024-12-09T17:09:34,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741844_1020 (size=12509)
2024-12-09T17:09:34,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741844_1020 (size=12509)
2024-12-09T17:09:34,652 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/56a145a4a3fd4e429a2667299d1c88a7
2024-12-09T17:09:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/56a145a4a3fd4e429a2667299d1c88a7 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/56a145a4a3fd4e429a2667299d1c88a7
2024-12-09T17:09:34,663 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/56a145a4a3fd4e429a2667299d1c88a7, entries=7, sequenceid=84, filesize=12.2 K
2024-12-09T17:09:34,664 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 9b1c820bb7effbbb64e39968a9dba25e in 25ms, sequenceid=84, compaction requested=true
2024-12-09T17:09:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=85.2 K, sizeToCheck=16.0 K
2024-12-09T17:09:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:09:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8 because midkey is the same as first or last row
2024-12-09T17:09:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b1c820bb7effbbb64e39968a9dba25e:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:09:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:34,664 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:09:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:34,665 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-09T17:09:34,665 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 87248 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:09:34,666 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): 9b1c820bb7effbbb64e39968a9dba25e/info is initiating minor compaction (all files)
2024-12-09T17:09:34,666 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b1c820bb7effbbb64e39968a9dba25e/info in TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:34,666 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/8d682c258b514256b002f37d58973893, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/56a145a4a3fd4e429a2667299d1c88a7] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp, totalSize=85.2 K
2024-12-09T17:09:34,666 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 02b61623b0db4d5a9a51060795793cd8, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733764150355
2024-12-09T17:09:34,667 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8d682c258b514256b002f37d58973893, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733764162502
2024-12-09T17:09:34,667 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 56a145a4a3fd4e429a2667299d1c88a7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733764172624
2024-12-09T17:09:34,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/a4aa5277be08496a81a692657f033334 is 1080, key is row0068/info:/1733764174640/Put/seqid=0
2024-12-09T17:09:34,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741845_1021 (size=24376)
2024-12-09T17:09:34,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741845_1021 (size=24376)
2024-12-09T17:09:34,682 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b1c820bb7effbbb64e39968a9dba25e#info#compaction#48 average throughput is 22.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:09:34,682 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/d167ce7c925b4ad289d96e4f1049e0f0 is 1080, key is row0001/info:/1733764150355/Put/seqid=0
2024-12-09T17:09:34,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741846_1022 (size=77532)
2024-12-09T17:09:34,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741846_1022 (size=77532)
2024-12-09T17:09:34,695 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/d167ce7c925b4ad289d96e4f1049e0f0 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0
2024-12-09T17:09:34,701 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b1c820bb7effbbb64e39968a9dba25e/info of 9b1c820bb7effbbb64e39968a9dba25e into d167ce7c925b4ad289d96e4f1049e0f0(size=75.7 K), total size for store is 75.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:09:34,701 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:34,701 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e., storeName=9b1c820bb7effbbb64e39968a9dba25e/info, priority=13, startTime=1733764174664; duration=0sec
2024-12-09T17:09:34,701 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=75.7 K, sizeToCheck=16.0 K
2024-12-09T17:09:34,701 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-09T17:09:34,702 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:34,702 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:34,702 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b1c820bb7effbbb64e39968a9dba25e:info
2024-12-09T17:09:34,703 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46085 {}] assignment.AssignmentManager(1346): Split request from 80c69eb3c456,43437,1733764138692, parent={ENCODED => 9b1c820bb7effbbb64e39968a9dba25e, NAME => 'TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-09T17:09:34,707 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46085 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:34,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46085 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9b1c820bb7effbbb64e39968a9dba25e, daughterA=b03d2b25b40018e2c8bdcbc90f872efe, daughterB=e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:34,711 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9b1c820bb7effbbb64e39968a9dba25e, daughterA=b03d2b25b40018e2c8bdcbc90f872efe, daughterB=e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:34,711 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9b1c820bb7effbbb64e39968a9dba25e, daughterA=b03d2b25b40018e2c8bdcbc90f872efe, daughterB=e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:34,711 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9b1c820bb7effbbb64e39968a9dba25e, daughterA=b03d2b25b40018e2c8bdcbc90f872efe, daughterB=e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:34,717 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, UNASSIGN}]
2024-12-09T17:09:34,718 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, UNASSIGN
2024-12-09T17:09:34,719 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=9b1c820bb7effbbb64e39968a9dba25e, regionState=CLOSING, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:34,721 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-09T17:09:34,721 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; CloseRegionProcedure 9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692}]
2024-12-09T17:09:34,878 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,43437,1733764138692
2024-12-09T17:09:34,879 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(124): Close 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:34,879 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(138): Unassign region: split region: true: evictCache: true
2024-12-09T17:09:34,880 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1681): Closing 9b1c820bb7effbbb64e39968a9dba25e, disabling compactions & flushes
2024-12-09T17:09:34,880 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1942): waiting for 0 compactions & cache flush to complete for region TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:34,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:35,077 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/a4aa5277be08496a81a692657f033334
2024-12-09T17:09:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/a4aa5277be08496a81a692657f033334 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/a4aa5277be08496a81a692657f033334
2024-12-09T17:09:35,097 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/a4aa5277be08496a81a692657f033334, entries=18, sequenceid=105, filesize=23.8 K
2024-12-09T17:09:35,097 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=11.56 KB/11836 for 9b1c820bb7effbbb64e39968a9dba25e in 432ms, sequenceid=105, compaction requested=false
2024-12-09T17:09:35,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:35,097 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:35,098 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:35,098 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e. after waiting 0 ms
2024-12-09T17:09:35,098 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:35,098 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(2837): Flushing 9b1c820bb7effbbb64e39968a9dba25e 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-09T17:09:35,101 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/0241231ac88747439bc15e8e44c580c8 is 1080, key is row0086/info:/1733764174665/Put/seqid=0
2024-12-09T17:09:35,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741847_1023 (size=16817)
2024-12-09T17:09:35,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741847_1023 (size=16817)
2024-12-09T17:09:35,106 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/0241231ac88747439bc15e8e44c580c8
2024-12-09T17:09:35,112 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/.tmp/info/0241231ac88747439bc15e8e44c580c8 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/0241231ac88747439bc15e8e44c580c8
2024-12-09T17:09:35,118 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/0241231ac88747439bc15e8e44c580c8, entries=11, sequenceid=120, filesize=16.4 K
2024-12-09T17:09:35,119 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(3040): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 9b1c820bb7effbbb64e39968a9dba25e in 21ms, sequenceid=120, compaction requested=true
2024-12-09T17:09:35,120 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/7c1882986707431a9b27b5311bd4bf5c, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/12c6f2f7dc20416cbdb13ae82c2adb79, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/8d682c258b514256b002f37d58973893, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/56a145a4a3fd4e429a2667299d1c88a7] to archive
2024-12-09T17:09:35,121 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-09T17:09:35,122 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/7c1882986707431a9b27b5311bd4bf5c to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/7c1882986707431a9b27b5311bd4bf5c
2024-12-09T17:09:35,124 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/956cadc4b48b4766acdf99f99477bdf9
2024-12-09T17:09:35,125 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/02b61623b0db4d5a9a51060795793cd8
2024-12-09T17:09:35,126 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/12c6f2f7dc20416cbdb13ae82c2adb79 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/12c6f2f7dc20416cbdb13ae82c2adb79
2024-12-09T17:09:35,127 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/8d682c258b514256b002f37d58973893 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/8d682c258b514256b002f37d58973893
2024-12-09T17:09:35,128 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/56a145a4a3fd4e429a2667299d1c88a7 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/56a145a4a3fd4e429a2667299d1c88a7
2024-12-09T17:09:35,132 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/recovered.edits/123.seqid, newMaxSeqId=123, maxSeqId=1
2024-12-09T17:09:35,132 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.
2024-12-09T17:09:35,132 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1635): Region close journal for 9b1c820bb7effbbb64e39968a9dba25e:

2024-12-09T17:09:35,134 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(170): Closed 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,134 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=9b1c820bb7effbbb64e39968a9dba25e, regionState=CLOSED
2024-12-09T17:09:35,137 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13
2024-12-09T17:09:35,137 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; CloseRegionProcedure 9b1c820bb7effbbb64e39968a9dba25e, server=80c69eb3c456,43437,1733764138692 in 415 msec
2024-12-09T17:09:35,138 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12
2024-12-09T17:09:35,138 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b1c820bb7effbbb64e39968a9dba25e, UNASSIGN in 420 msec
2024-12-09T17:09:35,154 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:35,155 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=12 splitting 3 storefiles, region=9b1c820bb7effbbb64e39968a9dba25e, threads=3
2024-12-09T17:09:35,156 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/0241231ac88747439bc15e8e44c580c8 for region: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,156 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/a4aa5277be08496a81a692657f033334 for region: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,156 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0 for region: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,164 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/0241231ac88747439bc15e8e44c580c8, top=true
2024-12-09T17:09:35,165 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/a4aa5277be08496a81a692657f033334, top=true
2024-12-09T17:09:35,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741848_1024 (size=27)
2024-12-09T17:09:35,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741848_1024 (size=27)
2024-12-09T17:09:35,176 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8 for child: e187c0a2092249527a1f31b04e9ee7c2, parent: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,176 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334 for child: e187c0a2092249527a1f31b04e9ee7c2, parent: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,176 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/a4aa5277be08496a81a692657f033334 for region: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,176 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/0241231ac88747439bc15e8e44c580c8 for region: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741849_1025 (size=27)
2024-12-09T17:09:35,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741849_1025 (size=27)
2024-12-09T17:09:35,180 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0 for region: 9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:09:35,180 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=12 split storefiles for region 9b1c820bb7effbbb64e39968a9dba25e Daughter A: [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e] storefiles, Daughter B: [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e] storefiles.
2024-12-09T17:09:35,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741850_1026 (size=71)
2024-12-09T17:09:35,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741850_1026 (size=71)
2024-12-09T17:09:35,190 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:35,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741851_1027 (size=71)
2024-12-09T17:09:35,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741851_1027 (size=71)
2024-12-09T17:09:35,203 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:35,213 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/recovered.edits/123.seqid, newMaxSeqId=123, maxSeqId=-1
2024-12-09T17:09:35,215 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/recovered.edits/123.seqid, newMaxSeqId=123, maxSeqId=-1
2024-12-09T17:09:35,217 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733764175216"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733764175216"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733764175216"}]},"ts":"1733764175216"}
2024-12-09T17:09:35,217 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733764175216"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764175216"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733764175216"}]},"ts":"1733764175216"}
2024-12-09T17:09:35,217 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733764175216"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764175216"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733764175216"}]},"ts":"1733764175216"}
2024-12-09T17:09:35,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43437 {}] regionserver.HRegion(8581): Flush requested on 1588230740
2024-12-09T17:09:35,243 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-09T17:09:35,243 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=4.75 KB heapSize=8.29 KB
2024-12-09T17:09:35,247 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b03d2b25b40018e2c8bdcbc90f872efe, ASSIGN}, {pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e187c0a2092249527a1f31b04e9ee7c2, ASSIGN}]
2024-12-09T17:09:35,249 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e187c0a2092249527a1f31b04e9ee7c2, ASSIGN
2024-12-09T17:09:35,249 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b03d2b25b40018e2c8bdcbc90f872efe, ASSIGN
2024-12-09T17:09:35,249 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e187c0a2092249527a1f31b04e9ee7c2, ASSIGN; state=SPLITTING_NEW, location=80c69eb3c456,43437,1733764138692; forceNewPlan=false, retain=false
2024-12-09T17:09:35,249 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b03d2b25b40018e2c8bdcbc90f872efe, ASSIGN; state=SPLITTING_NEW, location=80c69eb3c456,43437,1733764138692; forceNewPlan=false, retain=false
2024-12-09T17:09:35,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/info/dddfcc70c4d849b199152b980ca70d4d is 193, key is TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2./info:regioninfo/1733764175216/Put/seqid=0
2024-12-09T17:09:35,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741852_1028 (size=9423)
2024-12-09T17:09:35,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741852_1028 (size=9423)
2024-12-09T17:09:35,264 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.54 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/info/dddfcc70c4d849b199152b980ca70d4d
2024-12-09T17:09:35,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/table/96628b10b72745ba8e0deec391ac0f1a is 65, key is TestLogRolling-testLogRolling/table:state/1733764140720/Put/seqid=0
2024-12-09T17:09:35,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741853_1029 (size=5412)
2024-12-09T17:09:35,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741853_1029 (size=5412)
2024-12-09T17:09:35,286 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=216 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/table/96628b10b72745ba8e0deec391ac0f1a
2024-12-09T17:09:35,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/info/dddfcc70c4d849b199152b980ca70d4d as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/info/dddfcc70c4d849b199152b980ca70d4d
2024-12-09T17:09:35,295 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/info/dddfcc70c4d849b199152b980ca70d4d, entries=29, sequenceid=17, filesize=9.2 K
2024-12-09T17:09:35,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/table/96628b10b72745ba8e0deec391ac0f1a as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/table/96628b10b72745ba8e0deec391ac0f1a
2024-12-09T17:09:35,301 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/table/96628b10b72745ba8e0deec391ac0f1a, entries=4, sequenceid=17, filesize=5.3 K
2024-12-09T17:09:35,301 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.75 KB/4869, heapSize ~8.01 KB/8200, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=17, compaction requested=false
2024-12-09T17:09:35,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-09T17:09:35,400 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=e187c0a2092249527a1f31b04e9ee7c2, regionState=OPENING, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:35,400 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=b03d2b25b40018e2c8bdcbc90f872efe, regionState=OPENING, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:35,402 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692}]
2024-12-09T17:09:35,404 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=15, state=RUNNABLE; OpenRegionProcedure b03d2b25b40018e2c8bdcbc90f872efe, server=80c69eb3c456,43437,1733764138692}]
2024-12-09T17:09:35,556 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,43437,1733764138692
2024-12-09T17:09:35,563 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:35,564 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => e187c0a2092249527a1f31b04e9ee7c2, NAME => 'TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-09T17:09:35,564 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:35,564 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:09:35,564 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:35,564 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:35,567 INFO  [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e187c0a2092249527a1f31b04e9ee7c2 
2024-12-09T17:09:35,568 INFO  [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e187c0a2092249527a1f31b04e9ee7c2 columnFamilyName info
2024-12-09T17:09:35,568 DEBUG [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:35,581 DEBUG [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8
2024-12-09T17:09:35,584 DEBUG [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334
2024-12-09T17:09:35,591 DEBUG [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e->hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0-top
2024-12-09T17:09:35,591 INFO  [StoreOpener-e187c0a2092249527a1f31b04e9ee7c2-1 {}] regionserver.HStore(327): Store=e187c0a2092249527a1f31b04e9ee7c2/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:09:35,592 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:35,593 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:35,595 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:35,596 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1102): Opened e187c0a2092249527a1f31b04e9ee7c2; next sequenceid=124; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787562, jitterRate=0.0014369040727615356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:09:35,596 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:35,597 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., pid=17, masterSystemTime=1733764175556
2024-12-09T17:09:35,597 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:09:35,597 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:35,597 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:09:35,599 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:35,599 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:09:35,599 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:35,599 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e->hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0-top, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=115.9 K
2024-12-09T17:09:35,599 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:35,599 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:35,599 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:09:35,599 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7285): Opening region: {ENCODED => b03d2b25b40018e2c8bdcbc90f872efe, NAME => 'TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-09T17:09:35,599 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=e187c0a2092249527a1f31b04e9ee7c2, regionState=OPEN, openSeqNum=124, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:35,599 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.Compactor(224): Compacting d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733764150355
2024-12-09T17:09:35,600 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:09:35,600 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:09:35,600 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7327): checking encryption for b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:09:35,600 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7330): checking classloading for b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:09:35,600 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733764174640
2024-12-09T17:09:35,600 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733764174665
2024-12-09T17:09:35,601 INFO  [StoreOpener-b03d2b25b40018e2c8bdcbc90f872efe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b03d2b25b40018e2c8bdcbc90f872efe 
2024-12-09T17:09:35,602 INFO  [StoreOpener-b03d2b25b40018e2c8bdcbc90f872efe-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b03d2b25b40018e2c8bdcbc90f872efe columnFamilyName info
2024-12-09T17:09:35,602 DEBUG [StoreOpener-b03d2b25b40018e2c8bdcbc90f872efe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:09:35,603 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16
2024-12-09T17:09:35,603 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692 in 199 msec
2024-12-09T17:09:35,604 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e187c0a2092249527a1f31b04e9ee7c2, ASSIGN in 356 msec
2024-12-09T17:09:35,609 DEBUG [StoreOpener-b03d2b25b40018e2c8bdcbc90f872efe-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e->hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0-bottom
2024-12-09T17:09:35,610 INFO  [StoreOpener-b03d2b25b40018e2c8bdcbc90f872efe-1 {}] regionserver.HStore(327): Store=b03d2b25b40018e2c8bdcbc90f872efe/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:09:35,611 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:09:35,612 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:09:35,614 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1085): writing seq id for b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:09:35,614 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1102): Opened b03d2b25b40018e2c8bdcbc90f872efe; next sequenceid=124; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709243, jitterRate=-0.09815186262130737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:09:35,615 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1001): Region open journal for b03d2b25b40018e2c8bdcbc90f872efe:

2024-12-09T17:09:35,615 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe., pid=18, masterSystemTime=1733764175556
2024-12-09T17:09:35,615 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(403): Add compact mark for store b03d2b25b40018e2c8bdcbc90f872efe:info, priority=-2147483648, current under compaction store size is 2
2024-12-09T17:09:35,615 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:35,615 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-09T17:09:35,616 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:09:35,616 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): b03d2b25b40018e2c8bdcbc90f872efe/info is initiating minor compaction (all files)
2024-12-09T17:09:35,616 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b03d2b25b40018e2c8bdcbc90f872efe/info in TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:09:35,616 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e->hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0-bottom] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/.tmp, totalSize=75.7 K
2024-12-09T17:09:35,617 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733764150355
2024-12-09T17:09:35,617 DEBUG [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:09:35,617 INFO  [RS_OPEN_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:09:35,618 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=b03d2b25b40018e2c8bdcbc90f872efe, regionState=OPEN, openSeqNum=124, regionLocation=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:35,621 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=15
2024-12-09T17:09:35,621 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=15, state=SUCCESS; OpenRegionProcedure b03d2b25b40018e2c8bdcbc90f872efe, server=80c69eb3c456,43437,1733764138692 in 216 msec
2024-12-09T17:09:35,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=12
2024-12-09T17:09:35,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b03d2b25b40018e2c8bdcbc90f872efe, ASSIGN in 374 msec
2024-12-09T17:09:35,623 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#52 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:09:35,623 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/8c3bb9ec2f0641d6a721558ea24cf142 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:09:35,624 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9b1c820bb7effbbb64e39968a9dba25e, daughterA=b03d2b25b40018e2c8bdcbc90f872efe, daughterB=e187c0a2092249527a1f31b04e9ee7c2 in 916 msec
2024-12-09T17:09:35,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741854_1030 (size=42984)
2024-12-09T17:09:35,630 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741854_1030 (size=42984)
2024-12-09T17:09:35,634 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b03d2b25b40018e2c8bdcbc90f872efe#info#compaction#53 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:09:35,635 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/.tmp/info/594c3e61a5904d8fa203cf268d5b214a is 1080, key is row0001/info:/1733764150355/Put/seqid=0
2024-12-09T17:09:35,635 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/8c3bb9ec2f0641d6a721558ea24cf142 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/8c3bb9ec2f0641d6a721558ea24cf142
2024-12-09T17:09:35,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741855_1031 (size=70862)
2024-12-09T17:09:35,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741855_1031 (size=70862)
2024-12-09T17:09:35,641 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into 8c3bb9ec2f0641d6a721558ea24cf142(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:09:35,641 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:35,641 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764175597; duration=0sec
2024-12-09T17:09:35,641 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:35,641 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:09:35,645 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/.tmp/info/594c3e61a5904d8fa203cf268d5b214a as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/594c3e61a5904d8fa203cf268d5b214a
2024-12-09T17:09:35,650 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 1 (all) file(s) in b03d2b25b40018e2c8bdcbc90f872efe/info of b03d2b25b40018e2c8bdcbc90f872efe into 594c3e61a5904d8fa203cf268d5b214a(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:09:35,650 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b03d2b25b40018e2c8bdcbc90f872efe:

2024-12-09T17:09:35,650 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe., storeName=b03d2b25b40018e2c8bdcbc90f872efe/info, priority=15, startTime=1733764175615; duration=0sec
2024-12-09T17:09:35,650 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:35,650 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b03d2b25b40018e2c8bdcbc90f872efe:info
2024-12-09T17:09:35,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60786 deadline: 1733764186685, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733764140336.9b1c820bb7effbbb64e39968a9dba25e. is not online on 80c69eb3c456,43437,1733764138692
2024-12-09T17:09:36,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:37,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:38,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:39,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:40,134 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,134 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,135 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,135 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,136 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,136 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,160 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,160 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,160 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,160 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,161 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,161 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,164 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,165 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,165 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,167 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,558 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-09T17:09:40,558 INFO  [master/80c69eb3c456:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-09T17:09:40,677 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:09:40,679 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,682 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,683 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,700 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,702 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,702 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,703 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,704 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:09:40,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:41,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:42,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:43,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:44,630 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835
2024-12-09T17:09:44,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:45,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:46,781 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:09:46,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/e077a696ffe347709f0f9ace76dd285f is 1080, key is row0097/info:/1733764186773/Put/seqid=0
2024-12-09T17:09:46,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741856_1032 (size=12515)
2024-12-09T17:09:46,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741856_1032 (size=12515)
2024-12-09T17:09:46,792 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/e077a696ffe347709f0f9ace76dd285f
2024-12-09T17:09:46,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/e077a696ffe347709f0f9ace76dd285f as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/e077a696ffe347709f0f9ace76dd285f
2024-12-09T17:09:46,802 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/e077a696ffe347709f0f9ace76dd285f, entries=7, sequenceid=134, filesize=12.2 K
2024-12-09T17:09:46,803 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for e187c0a2092249527a1f31b04e9ee7c2 in 22ms, sequenceid=134, compaction requested=false
2024-12-09T17:09:46,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:46,804 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-09T17:09:46,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/430d3fcefd6b4c1781049c00447e53a4 is 1080, key is row0104/info:/1733764186781/Put/seqid=0
2024-12-09T17:09:46,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741857_1033 (size=15750)
2024-12-09T17:09:46,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741857_1033 (size=15750)
2024-12-09T17:09:46,822 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/430d3fcefd6b4c1781049c00447e53a4
2024-12-09T17:09:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/430d3fcefd6b4c1781049c00447e53a4 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/430d3fcefd6b4c1781049c00447e53a4
2024-12-09T17:09:46,835 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/430d3fcefd6b4c1781049c00447e53a4, entries=10, sequenceid=147, filesize=15.4 K
2024-12-09T17:09:46,836 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=15.76 KB/16140 for e187c0a2092249527a1f31b04e9ee7c2 in 32ms, sequenceid=147, compaction requested=true
2024-12-09T17:09:46,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:46,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:09:46,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:46,836 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:09:46,837 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 71249 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:09:46,837 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:09:46,837 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:46,838 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/8c3bb9ec2f0641d6a721558ea24cf142, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/e077a696ffe347709f0f9ace76dd285f, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/430d3fcefd6b4c1781049c00447e53a4] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=69.6 K
2024-12-09T17:09:46,838 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8c3bb9ec2f0641d6a721558ea24cf142, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733764172627
2024-12-09T17:09:46,838 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting e077a696ffe347709f0f9ace76dd285f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733764186773
2024-12-09T17:09:46,839 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 430d3fcefd6b4c1781049c00447e53a4, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733764186781
2024-12-09T17:09:46,854 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#56 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:09:46,855 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/313a268c738e41aaa28a08826368a642 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:09:46,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741858_1034 (size=61415)
2024-12-09T17:09:46,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741858_1034 (size=61415)
2024-12-09T17:09:46,876 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/313a268c738e41aaa28a08826368a642 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/313a268c738e41aaa28a08826368a642
2024-12-09T17:09:46,883 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into 313a268c738e41aaa28a08826368a642(size=60.0 K), total size for store is 60.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:09:46,883 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:46,883 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764186836; duration=0sec
2024-12-09T17:09:46,884 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:46,884 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:09:46,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:47,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:48,827 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-12-09T17:09:48,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/3bb91e654a484203a0f8ad7951a02f9d is 1080, key is row0114/info:/1733764186804/Put/seqid=0
2024-12-09T17:09:48,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741859_1035 (size=22238)
2024-12-09T17:09:48,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741859_1035 (size=22238)
2024-12-09T17:09:48,837 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/3bb91e654a484203a0f8ad7951a02f9d
2024-12-09T17:09:48,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/3bb91e654a484203a0f8ad7951a02f9d as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/3bb91e654a484203a0f8ad7951a02f9d
2024-12-09T17:09:48,844 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-09T17:09:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60786 deadline: 1733764198844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:48,848 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/3bb91e654a484203a0f8ad7951a02f9d, entries=16, sequenceid=167, filesize=21.7 K
2024-12-09T17:09:48,849 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for e187c0a2092249527a1f31b04e9ee7c2 in 22ms, sequenceid=167, compaction requested=false
2024-12-09T17:09:48,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:48,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:49,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:50,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:51,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:52,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:53,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:54,975 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:55,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:56,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:57,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:58,533 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-09T17:09:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:09:58,944 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-09T17:09:58,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/7ff7253be7bb4156b349819cdc96dcdb is 1080, key is row0130/info:/1733764188828/Put/seqid=0
2024-12-09T17:09:58,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741860_1036 (size=20078)
2024-12-09T17:09:58,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741860_1036 (size=20078)
2024-12-09T17:09:58,961 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/7ff7253be7bb4156b349819cdc96dcdb
2024-12-09T17:09:58,965 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-09T17:09:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60786 deadline: 1733764208965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692
2024-12-09T17:09:58,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/7ff7253be7bb4156b349819cdc96dcdb as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/7ff7253be7bb4156b349819cdc96dcdb
2024-12-09T17:09:58,972 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/7ff7253be7bb4156b349819cdc96dcdb, entries=14, sequenceid=184, filesize=19.6 K
2024-12-09T17:09:58,973 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=15.76 KB/16140 for e187c0a2092249527a1f31b04e9ee7c2 in 28ms, sequenceid=184, compaction requested=true
2024-12-09T17:09:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:09:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:58,973 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:09:58,974 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 103731 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:09:58,974 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:09:58,974 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:09:58,974 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/313a268c738e41aaa28a08826368a642, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/3bb91e654a484203a0f8ad7951a02f9d, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/7ff7253be7bb4156b349819cdc96dcdb] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=101.3 K
2024-12-09T17:09:58,974 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 313a268c738e41aaa28a08826368a642, keycount=52, bloomtype=ROW, size=60.0 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733764172627
2024-12-09T17:09:58,975 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3bb91e654a484203a0f8ad7951a02f9d, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733764186804
2024-12-09T17:09:58,975 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7ff7253be7bb4156b349819cdc96dcdb, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733764188828
2024-12-09T17:09:58,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:09:58,986 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#59 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:09:58,986 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/4301ea08dba64061a5a71dc289025097 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:09:58,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741861_1037 (size=93998)
2024-12-09T17:09:58,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741861_1037 (size=93998)
2024-12-09T17:09:58,995 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/4301ea08dba64061a5a71dc289025097 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/4301ea08dba64061a5a71dc289025097
2024-12-09T17:09:59,000 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into 4301ea08dba64061a5a71dc289025097(size=91.8 K), total size for store is 91.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:09:59,000 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:09:59,000 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764198973; duration=0sec
2024-12-09T17:09:59,001 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:09:59,001 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:09:59,981 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:00,284 DEBUG [master/80c69eb3c456:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache
2024-12-09T17:10:00,285 DEBUG [master/80c69eb3c456:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region deb7453c97cf6e1b1675a89c6a7593bf changed from -1.0 to 0.0, refreshing cache
2024-12-09T17:10:00,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:01,983 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:02,984 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:03,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:04,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:05,986 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:06,987 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:07,988 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:08,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:09,008 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-12-09T17:10:09,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6813e0a0e7f24762bdea6e9d7982f4d4 is 1080, key is row0144/info:/1733764198944/Put/seqid=0
2024-12-09T17:10:09,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741862_1038 (size=22238)
2024-12-09T17:10:09,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741862_1038 (size=22238)
2024-12-09T17:10:09,018 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6813e0a0e7f24762bdea6e9d7982f4d4
2024-12-09T17:10:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6813e0a0e7f24762bdea6e9d7982f4d4 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6813e0a0e7f24762bdea6e9d7982f4d4
2024-12-09T17:10:09,029 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6813e0a0e7f24762bdea6e9d7982f4d4, entries=16, sequenceid=204, filesize=21.7 K
2024-12-09T17:10:09,030 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=1.05 KB/1076 for e187c0a2092249527a1f31b04e9ee7c2 in 21ms, sequenceid=204, compaction requested=false
2024-12-09T17:10:09,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:09,991 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:09,992 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta after 196224ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-09T17:10:10,993 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:11,028 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:10:11,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/68edeea6db17456b8706ec0e3373a700 is 1080, key is row0160/info:/1733764209009/Put/seqid=0
2024-12-09T17:10:11,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741863_1039 (size=12516)
2024-12-09T17:10:11,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741863_1039 (size=12516)
2024-12-09T17:10:11,042 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/68edeea6db17456b8706ec0e3373a700
2024-12-09T17:10:11,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/68edeea6db17456b8706ec0e3373a700 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/68edeea6db17456b8706ec0e3373a700
2024-12-09T17:10:11,055 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/68edeea6db17456b8706ec0e3373a700, entries=7, sequenceid=214, filesize=12.2 K
2024-12-09T17:10:11,056 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for e187c0a2092249527a1f31b04e9ee7c2 in 28ms, sequenceid=214, compaction requested=true
2024-12-09T17:10:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:10:11,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:11,056 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:10:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:11,057 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-12-09T17:10:11,057 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128752 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:10:11,057 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:10:11,058 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:11,058 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/4301ea08dba64061a5a71dc289025097, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6813e0a0e7f24762bdea6e9d7982f4d4, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/68edeea6db17456b8706ec0e3373a700] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=125.7 K
2024-12-09T17:10:11,058 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4301ea08dba64061a5a71dc289025097, keycount=82, bloomtype=ROW, size=91.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733764172627
2024-12-09T17:10:11,058 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6813e0a0e7f24762bdea6e9d7982f4d4, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733764198944
2024-12-09T17:10:11,059 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 68edeea6db17456b8706ec0e3373a700, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733764209009
2024-12-09T17:10:11,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/76165a61c76a4f87a5603576d8a794d0 is 1080, key is row0167/info:/1733764211029/Put/seqid=0
2024-12-09T17:10:11,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741864_1040 (size=25472)
2024-12-09T17:10:11,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741864_1040 (size=25472)
2024-12-09T17:10:11,073 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#63 average throughput is 53.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:10:11,073 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/ca0aacd913674ec5a2881cf0614fe541 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:10:11,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741865_1041 (size=118898)
2024-12-09T17:10:11,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741865_1041 (size=118898)
2024-12-09T17:10:11,082 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/ca0aacd913674ec5a2881cf0614fe541 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/ca0aacd913674ec5a2881cf0614fe541
2024-12-09T17:10:11,088 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into ca0aacd913674ec5a2881cf0614fe541(size=116.1 K), total size for store is 116.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:10:11,088 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:11,088 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764211056; duration=0sec
2024-12-09T17:10:11,088 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:11,088 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:10:11,471 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/76165a61c76a4f87a5603576d8a794d0
2024-12-09T17:10:11,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/76165a61c76a4f87a5603576d8a794d0 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/76165a61c76a4f87a5603576d8a794d0
2024-12-09T17:10:11,487 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/76165a61c76a4f87a5603576d8a794d0, entries=19, sequenceid=236, filesize=24.9 K
2024-12-09T17:10:11,488 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=7.36 KB/7532 for e187c0a2092249527a1f31b04e9ee7c2 in 431ms, sequenceid=236, compaction requested=false
2024-12-09T17:10:11,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:11,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:12,995 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:13,070 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-12-09T17:10:13,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6db28d910be346c5ba8e186f35c7d4a3 is 1080, key is row0186/info:/1733764211057/Put/seqid=0
2024-12-09T17:10:13,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741866_1042 (size=13594)
2024-12-09T17:10:13,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741866_1042 (size=13594)
2024-12-09T17:10:13,087 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6db28d910be346c5ba8e186f35c7d4a3
2024-12-09T17:10:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6db28d910be346c5ba8e186f35c7d4a3 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6db28d910be346c5ba8e186f35c7d4a3
2024-12-09T17:10:13,097 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6db28d910be346c5ba8e186f35c7d4a3, entries=8, sequenceid=248, filesize=13.3 K
2024-12-09T17:10:13,098 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=16.81 KB/17216 for e187c0a2092249527a1f31b04e9ee7c2 in 28ms, sequenceid=248, compaction requested=true
2024-12-09T17:10:13,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:13,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:10:13,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:13,098 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:10:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:13,099 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-12-09T17:10:13,100 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 157964 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:10:13,100 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:10:13,100 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:13,100 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/ca0aacd913674ec5a2881cf0614fe541, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/76165a61c76a4f87a5603576d8a794d0, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6db28d910be346c5ba8e186f35c7d4a3] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=154.3 K
2024-12-09T17:10:13,100 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting ca0aacd913674ec5a2881cf0614fe541, keycount=105, bloomtype=ROW, size=116.1 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733764172627
2024-12-09T17:10:13,101 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 76165a61c76a4f87a5603576d8a794d0, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733764211029
2024-12-09T17:10:13,101 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6db28d910be346c5ba8e186f35c7d4a3, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733764211057
2024-12-09T17:10:13,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/cef4fa7191704710a482e31dd1831343 is 1080, key is row0194/info:/1733764213071/Put/seqid=0
2024-12-09T17:10:13,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741867_1043 (size=23327)
2024-12-09T17:10:13,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741867_1043 (size=23327)
2024-12-09T17:10:13,108 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/cef4fa7191704710a482e31dd1831343
2024-12-09T17:10:13,114 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#66 average throughput is 67.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:10:13,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/cef4fa7191704710a482e31dd1831343 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cef4fa7191704710a482e31dd1831343
2024-12-09T17:10:13,115 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/f46e276bcfd94f91834e79dbaf16bcb8 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:10:13,116 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-09T17:10:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60786 deadline: 1733764223115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=e187c0a2092249527a1f31b04e9ee7c2, server=80c69eb3c456,43437,1733764138692
2024-12-09T17:10:13,119 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cef4fa7191704710a482e31dd1831343, entries=17, sequenceid=268, filesize=22.8 K
2024-12-09T17:10:13,120 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=12.61 KB/12912 for e187c0a2092249527a1f31b04e9ee7c2 in 21ms, sequenceid=268, compaction requested=false
2024-12-09T17:10:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:13,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741868_1044 (size=148311)
2024-12-09T17:10:13,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741868_1044 (size=148311)
2024-12-09T17:10:13,126 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/f46e276bcfd94f91834e79dbaf16bcb8 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/f46e276bcfd94f91834e79dbaf16bcb8
2024-12-09T17:10:13,131 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into f46e276bcfd94f91834e79dbaf16bcb8(size=144.8 K), total size for store is 167.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:10:13,131 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:13,131 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764213098; duration=0sec
2024-12-09T17:10:13,131 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:13,132 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:10:13,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:14,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:15,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:16,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:17,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:19,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:20,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:20,564 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region e187c0a2092249527a1f31b04e9ee7c2, had cached 0 bytes from a total of 171638
2024-12-09T17:10:20,600 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region b03d2b25b40018e2c8bdcbc90f872efe, had cached 0 bytes from a total of 70862
2024-12-09T17:10:21,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:22,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:23,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:23,167 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-09T17:10:23,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6ab501e1b7ec4f84a5963a132b0fb995 is 1080, key is row0211/info:/1733764213100/Put/seqid=0
2024-12-09T17:10:23,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741869_1045 (size=19013)
2024-12-09T17:10:23,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741869_1045 (size=19013)
2024-12-09T17:10:23,183 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6ab501e1b7ec4f84a5963a132b0fb995
2024-12-09T17:10:23,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/6ab501e1b7ec4f84a5963a132b0fb995 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6ab501e1b7ec4f84a5963a132b0fb995
2024-12-09T17:10:23,194 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6ab501e1b7ec4f84a5963a132b0fb995, entries=13, sequenceid=285, filesize=18.6 K
2024-12-09T17:10:23,195 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=1.05 KB/1076 for e187c0a2092249527a1f31b04e9ee7c2 in 28ms, sequenceid=285, compaction requested=true
2024-12-09T17:10:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:10:23,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:23,196 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:10:23,197 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 190651 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:10:23,197 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:10:23,197 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:23,197 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/f46e276bcfd94f91834e79dbaf16bcb8, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cef4fa7191704710a482e31dd1831343, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6ab501e1b7ec4f84a5963a132b0fb995] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=186.2 K
2024-12-09T17:10:23,198 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting f46e276bcfd94f91834e79dbaf16bcb8, keycount=132, bloomtype=ROW, size=144.8 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733764172627
2024-12-09T17:10:23,198 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting cef4fa7191704710a482e31dd1831343, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733764213071
2024-12-09T17:10:23,199 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6ab501e1b7ec4f84a5963a132b0fb995, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733764213100
2024-12-09T17:10:23,213 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#68 average throughput is 55.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:10:23,214 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/cdcfa880ad054425beddb38d0f6a6797 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:10:23,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741870_1046 (size=180785)
2024-12-09T17:10:23,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741870_1046 (size=180785)
2024-12-09T17:10:23,227 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/cdcfa880ad054425beddb38d0f6a6797 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cdcfa880ad054425beddb38d0f6a6797
2024-12-09T17:10:23,234 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into cdcfa880ad054425beddb38d0f6a6797(size=176.5 K), total size for store is 176.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:10:23,234 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:23,234 INFO  [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764223196; duration=0sec
2024-12-09T17:10:23,234 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:23,234 DEBUG [RS:0;80c69eb3c456:43437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:10:24,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:25,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:25,176 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-09T17:10:25,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/91b7d0962b444620b440d02847c1ce57 is 1080, key is row0224/info:/1733764223168/Put/seqid=0
2024-12-09T17:10:25,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741871_1047 (size=12523)
2024-12-09T17:10:25,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741871_1047 (size=12523)
2024-12-09T17:10:25,193 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/91b7d0962b444620b440d02847c1ce57
2024-12-09T17:10:25,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/91b7d0962b444620b440d02847c1ce57 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/91b7d0962b444620b440d02847c1ce57
2024-12-09T17:10:25,204 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/91b7d0962b444620b440d02847c1ce57, entries=7, sequenceid=296, filesize=12.2 K
2024-12-09T17:10:25,205 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=22.07 KB/22596 for e187c0a2092249527a1f31b04e9ee7c2 in 29ms, sequenceid=296, compaction requested=false
2024-12-09T17:10:25,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43437 {}] regionserver.HRegion(8581): Flush requested on e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:25,205 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=23.12 KB heapSize=25 KB
2024-12-09T17:10:25,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/9d2734d8d82f40d4977be525be3ceb07 is 1080, key is row0231/info:/1733764225177/Put/seqid=0
2024-12-09T17:10:25,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741872_1048 (size=28728)
2024-12-09T17:10:25,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741872_1048 (size=28728)
2024-12-09T17:10:25,214 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/9d2734d8d82f40d4977be525be3ceb07
2024-12-09T17:10:25,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/9d2734d8d82f40d4977be525be3ceb07 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/9d2734d8d82f40d4977be525be3ceb07
2024-12-09T17:10:25,223 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/9d2734d8d82f40d4977be525be3ceb07, entries=22, sequenceid=321, filesize=28.1 K
2024-12-09T17:10:25,224 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=4.20 KB/4304 for e187c0a2092249527a1f31b04e9ee7c2 in 19ms, sequenceid=321, compaction requested=true
2024-12-09T17:10:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e187c0a2092249527a1f31b04e9ee7c2:info, priority=-2147483648, current under compaction store size is 1
2024-12-09T17:10:25,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:25,224 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-09T17:10:25,225 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 222036 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-09T17:10:25,225 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1540): e187c0a2092249527a1f31b04e9ee7c2/info is initiating minor compaction (all files)
2024-12-09T17:10:25,225 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e187c0a2092249527a1f31b04e9ee7c2/info in TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:25,225 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cdcfa880ad054425beddb38d0f6a6797, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/91b7d0962b444620b440d02847c1ce57, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/9d2734d8d82f40d4977be525be3ceb07] into tmpdir=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp, totalSize=216.8 K
2024-12-09T17:10:25,225 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.Compactor(224): Compacting cdcfa880ad054425beddb38d0f6a6797, keycount=162, bloomtype=ROW, size=176.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733764172627
2024-12-09T17:10:25,226 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.Compactor(224): Compacting 91b7d0962b444620b440d02847c1ce57, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733764223168
2024-12-09T17:10:25,226 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] compactions.Compactor(224): Compacting 9d2734d8d82f40d4977be525be3ceb07, keycount=22, bloomtype=ROW, size=28.1 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733764225177
2024-12-09T17:10:25,237 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e187c0a2092249527a1f31b04e9ee7c2#info#compaction#71 average throughput is 65.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-09T17:10:25,238 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/ebf2077543924aaba43dcc03051c5731 is 1080, key is row0062/info:/1733764172627/Put/seqid=0
2024-12-09T17:10:25,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741873_1049 (size=212255)
2024-12-09T17:10:25,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741873_1049 (size=212255)
2024-12-09T17:10:25,246 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/ebf2077543924aaba43dcc03051c5731 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/ebf2077543924aaba43dcc03051c5731
2024-12-09T17:10:25,251 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e187c0a2092249527a1f31b04e9ee7c2/info of e187c0a2092249527a1f31b04e9ee7c2 into ebf2077543924aaba43dcc03051c5731(size=207.3 K), total size for store is 207.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-09T17:10:25,251 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:25,251 INFO  [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., storeName=e187c0a2092249527a1f31b04e9ee7c2/info, priority=13, startTime=1733764225224; duration=0sec
2024-12-09T17:10:25,251 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-09T17:10:25,251 DEBUG [RS:0;80c69eb3c456:43437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e187c0a2092249527a1f31b04e9ee7c2:info
2024-12-09T17:10:26,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:27,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:27,210 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-09T17:10:27,211 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C43437%2C1733764138692.1733764227210
2024-12-09T17:10:27,219 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764139207 with entries=311, filesize=307.59 KB; new WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764227210
2024-12-09T17:10:27,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41693:41693),(127.0.0.1/127.0.0.1:36441:36441)]
2024-12-09T17:10:27,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764139207 is not closed yet, will try archiving it next time
2024-12-09T17:10:27,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741833_1009 (size=314980)
2024-12-09T17:10:27,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741833_1009 (size=314980)
2024-12-09T17:10:27,223 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing deb7453c97cf6e1b1675a89c6a7593bf 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-09T17:10:27,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/.tmp/info/68daf12593f5435ab341922d698102f6 is 45, key is default/info:d/1733764140134/Put/seqid=0
2024-12-09T17:10:27,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741875_1051 (size=5037)
2024-12-09T17:10:27,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741875_1051 (size=5037)
2024-12-09T17:10:27,241 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/.tmp/info/68daf12593f5435ab341922d698102f6
2024-12-09T17:10:27,247 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/.tmp/info/68daf12593f5435ab341922d698102f6 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/info/68daf12593f5435ab341922d698102f6
2024-12-09T17:10:27,252 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/info/68daf12593f5435ab341922d698102f6, entries=2, sequenceid=6, filesize=4.9 K
2024-12-09T17:10:27,253 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for deb7453c97cf6e1b1675a89c6a7593bf in 30ms, sequenceid=6, compaction requested=false
2024-12-09T17:10:27,254 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for deb7453c97cf6e1b1675a89c6a7593bf:

2024-12-09T17:10:27,254 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing e187c0a2092249527a1f31b04e9ee7c2 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-09T17:10:27,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/de99332294cc4509822c65dbc6ab05ba is 1080, key is row0253/info:/1733764225206/Put/seqid=0
2024-12-09T17:10:27,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741876_1052 (size=9278)
2024-12-09T17:10:27,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741876_1052 (size=9278)
2024-12-09T17:10:27,263 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/de99332294cc4509822c65dbc6ab05ba
2024-12-09T17:10:27,267 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/.tmp/info/de99332294cc4509822c65dbc6ab05ba as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/de99332294cc4509822c65dbc6ab05ba
2024-12-09T17:10:27,271 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/de99332294cc4509822c65dbc6ab05ba, entries=4, sequenceid=329, filesize=9.1 K
2024-12-09T17:10:27,271 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for e187c0a2092249527a1f31b04e9ee7c2 in 17ms, sequenceid=329, compaction requested=false
2024-12-09T17:10:27,272 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:27,272 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.21 KB heapSize=4.13 KB
2024-12-09T17:10:27,275 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/info/1d24d6a491b44db49f9fd94460cea33b is 193, key is TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2./info:regioninfo/1733764175599/Put/seqid=0
2024-12-09T17:10:27,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741877_1053 (size=7803)
2024-12-09T17:10:27,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741877_1053 (size=7803)
2024-12-09T17:10:27,279 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.21 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/info/1d24d6a491b44db49f9fd94460cea33b
2024-12-09T17:10:27,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/.tmp/info/1d24d6a491b44db49f9fd94460cea33b as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/info/1d24d6a491b44db49f9fd94460cea33b
2024-12-09T17:10:27,288 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/info/1d24d6a491b44db49f9fd94460cea33b, entries=16, sequenceid=24, filesize=7.6 K
2024-12-09T17:10:27,288 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~2.21 KB/2260, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 16ms, sequenceid=24, compaction requested=false
2024-12-09T17:10:27,288 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-09T17:10:27,289 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for b03d2b25b40018e2c8bdcbc90f872efe:

2024-12-09T17:10:27,289 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C43437%2C1733764138692.1733764227289
2024-12-09T17:10:27,294 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764227210 with entries=4, filesize=1.22 KB; new WAL /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764227289
2024-12-09T17:10:27,295 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41693:41693),(127.0.0.1/127.0.0.1:36441:36441)]
2024-12-09T17:10:27,295 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764139207 is not closed yet, will try archiving it next time
2024-12-09T17:10:27,295 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764227210 is not closed yet, will try archiving it next time
2024-12-09T17:10:27,295 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-09T17:10:27,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741874_1050 (size=1255)
2024-12-09T17:10:27,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741874_1050 (size=1255)
2024-12-09T17:10:27,297 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764139207 is not closed yet, will try archiving it next time
2024-12-09T17:10:27,297 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764227210 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/oldWALs/80c69eb3c456%2C43437%2C1733764138692.1733764227210
2024-12-09T17:10:27,624 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692/80c69eb3c456%2C43437%2C1733764138692.1733764139207 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/oldWALs/80c69eb3c456%2C43437%2C1733764138692.1733764139207
2024-12-09T17:10:27,696 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-09T17:10:27,696 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-09T17:10:27,696 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b07f769 to 127.0.0.1:49603
2024-12-09T17:10:27,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:27,697 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-09T17:10:27,697 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1854232385, stopped=false
2024-12-09T17:10:27,697 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=80c69eb3c456,46085,1733764138548
2024-12-09T17:10:27,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:10:27,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:10:27,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:27,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:27,740 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-09T17:10:27,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:27,741 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,43437,1733764138692' *****
2024-12-09T17:10:27,741 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:10:27,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:10:27,742 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:10:27,742 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:10:27,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:10:27,743 INFO  [RS:0;80c69eb3c456:43437 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:10:27,743 INFO  [RS:0;80c69eb3c456:43437 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:10:27,743 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(3579): Received CLOSE for deb7453c97cf6e1b1675a89c6a7593bf
2024-12-09T17:10:27,744 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(3579): Received CLOSE for e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:27,744 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing deb7453c97cf6e1b1675a89c6a7593bf, disabling compactions & flushes
2024-12-09T17:10:27,744 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(3579): Received CLOSE for b03d2b25b40018e2c8bdcbc90f872efe
2024-12-09T17:10:27,744 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,43437,1733764138692
2024-12-09T17:10:27,744 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:10:27,744 DEBUG [RS:0;80c69eb3c456:43437 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:27,744 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:10:27,744 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf. after waiting 0 ms
2024-12-09T17:10:27,744 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:10:27,744 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:10:27,744 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:10:27,744 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:10:27,745 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:10:27,745 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close
2024-12-09T17:10:27,745 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1603): Online Regions={deb7453c97cf6e1b1675a89c6a7593bf=hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf., e187c0a2092249527a1f31b04e9ee7c2=TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2., 1588230740=hbase:meta,,1.1588230740, b03d2b25b40018e2c8bdcbc90f872efe=TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.}
2024-12-09T17:10:27,745 DEBUG [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, b03d2b25b40018e2c8bdcbc90f872efe, deb7453c97cf6e1b1675a89c6a7593bf, e187c0a2092249527a1f31b04e9ee7c2
2024-12-09T17:10:27,746 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:10:27,746 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:10:27,746 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:10:27,746 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:10:27,746 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/namespace/deb7453c97cf6e1b1675a89c6a7593bf/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/hbase/meta/1588230740/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1
2024-12-09T17:10:27,750 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for deb7453c97cf6e1b1675a89c6a7593bf:

2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-09T17:10:27,750 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733764139694.deb7453c97cf6e1b1675a89c6a7593bf.
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e187c0a2092249527a1f31b04e9ee7c2, disabling compactions & flushes
2024-12-09T17:10:27,750 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2. after waiting 0 ms
2024-12-09T17:10:27,750 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:27,751 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e->hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0-top, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/8c3bb9ec2f0641d6a721558ea24cf142, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/e077a696ffe347709f0f9ace76dd285f, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/313a268c738e41aaa28a08826368a642, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/430d3fcefd6b4c1781049c00447e53a4, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/3bb91e654a484203a0f8ad7951a02f9d, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/4301ea08dba64061a5a71dc289025097, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/7ff7253be7bb4156b349819cdc96dcdb, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6813e0a0e7f24762bdea6e9d7982f4d4, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/ca0aacd913674ec5a2881cf0614fe541, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/68edeea6db17456b8706ec0e3373a700, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/76165a61c76a4f87a5603576d8a794d0, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/f46e276bcfd94f91834e79dbaf16bcb8, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6db28d910be346c5ba8e186f35c7d4a3, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cef4fa7191704710a482e31dd1831343, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cdcfa880ad054425beddb38d0f6a6797, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6ab501e1b7ec4f84a5963a132b0fb995, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/91b7d0962b444620b440d02847c1ce57, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/9d2734d8d82f40d4977be525be3ceb07] to archive
2024-12-09T17:10:27,752 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-09T17:10:27,753 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:10:27,755 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-a4aa5277be08496a81a692657f033334
2024-12-09T17:10:27,756 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/8c3bb9ec2f0641d6a721558ea24cf142 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/8c3bb9ec2f0641d6a721558ea24cf142
2024-12-09T17:10:27,757 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/TestLogRolling-testLogRolling=9b1c820bb7effbbb64e39968a9dba25e-0241231ac88747439bc15e8e44c580c8
2024-12-09T17:10:27,758 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/e077a696ffe347709f0f9ace76dd285f to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/e077a696ffe347709f0f9ace76dd285f
2024-12-09T17:10:27,760 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/313a268c738e41aaa28a08826368a642 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/313a268c738e41aaa28a08826368a642
2024-12-09T17:10:27,761 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/430d3fcefd6b4c1781049c00447e53a4 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/430d3fcefd6b4c1781049c00447e53a4
2024-12-09T17:10:27,762 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/3bb91e654a484203a0f8ad7951a02f9d to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/3bb91e654a484203a0f8ad7951a02f9d
2024-12-09T17:10:27,763 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/4301ea08dba64061a5a71dc289025097 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/4301ea08dba64061a5a71dc289025097
2024-12-09T17:10:27,765 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/7ff7253be7bb4156b349819cdc96dcdb to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/7ff7253be7bb4156b349819cdc96dcdb
2024-12-09T17:10:27,766 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6813e0a0e7f24762bdea6e9d7982f4d4 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6813e0a0e7f24762bdea6e9d7982f4d4
2024-12-09T17:10:27,767 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/ca0aacd913674ec5a2881cf0614fe541 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/ca0aacd913674ec5a2881cf0614fe541
2024-12-09T17:10:27,769 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/68edeea6db17456b8706ec0e3373a700 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/68edeea6db17456b8706ec0e3373a700
2024-12-09T17:10:27,770 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/76165a61c76a4f87a5603576d8a794d0 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/76165a61c76a4f87a5603576d8a794d0
2024-12-09T17:10:27,772 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/f46e276bcfd94f91834e79dbaf16bcb8 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/f46e276bcfd94f91834e79dbaf16bcb8
2024-12-09T17:10:27,773 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6db28d910be346c5ba8e186f35c7d4a3 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6db28d910be346c5ba8e186f35c7d4a3
2024-12-09T17:10:27,774 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cef4fa7191704710a482e31dd1831343 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cef4fa7191704710a482e31dd1831343
2024-12-09T17:10:27,776 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cdcfa880ad054425beddb38d0f6a6797 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/cdcfa880ad054425beddb38d0f6a6797
2024-12-09T17:10:27,777 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6ab501e1b7ec4f84a5963a132b0fb995 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/6ab501e1b7ec4f84a5963a132b0fb995
2024-12-09T17:10:27,779 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/91b7d0962b444620b440d02847c1ce57 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/91b7d0962b444620b440d02847c1ce57
2024-12-09T17:10:27,780 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/9d2734d8d82f40d4977be525be3ceb07 to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/info/9d2734d8d82f40d4977be525be3ceb07
2024-12-09T17:10:27,787 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/e187c0a2092249527a1f31b04e9ee7c2/recovered.edits/332.seqid, newMaxSeqId=332, maxSeqId=123
2024-12-09T17:10:27,787 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:27,788 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e187c0a2092249527a1f31b04e9ee7c2:

2024-12-09T17:10:27,788 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733764174707.e187c0a2092249527a1f31b04e9ee7c2.
2024-12-09T17:10:27,788 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b03d2b25b40018e2c8bdcbc90f872efe, disabling compactions & flushes
2024-12-09T17:10:27,788 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:10:27,788 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:10:27,788 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe. after waiting 0 ms
2024-12-09T17:10:27,788 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:10:27,788 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e->hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/9b1c820bb7effbbb64e39968a9dba25e/info/d167ce7c925b4ad289d96e4f1049e0f0-bottom] to archive
2024-12-09T17:10:27,790 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-09T17:10:27,792 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e to hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/archive/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/info/d167ce7c925b4ad289d96e4f1049e0f0.9b1c820bb7effbbb64e39968a9dba25e
2024-12-09T17:10:27,797 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/data/default/TestLogRolling-testLogRolling/b03d2b25b40018e2c8bdcbc90f872efe/recovered.edits/128.seqid, newMaxSeqId=128, maxSeqId=123
2024-12-09T17:10:27,798 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:10:27,798 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b03d2b25b40018e2c8bdcbc90f872efe:

2024-12-09T17:10:27,798 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733764174707.b03d2b25b40018e2c8bdcbc90f872efe.
2024-12-09T17:10:27,946 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,43437,1733764138692; all regions closed.
2024-12-09T17:10:27,946 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692
2024-12-09T17:10:27,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741834_1010 (size=9351)
2024-12-09T17:10:27,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741834_1010 (size=9351)
2024-12-09T17:10:27,950 DEBUG [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/oldWALs
2024-12-09T17:10:27,950 INFO  [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C43437%2C1733764138692.meta:.meta(num 1733764139623)
2024-12-09T17:10:27,951 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/WALs/80c69eb3c456,43437,1733764138692
2024-12-09T17:10:27,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741878_1054 (size=1072)
2024-12-09T17:10:27,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741878_1054 (size=1072)
2024-12-09T17:10:27,955 DEBUG [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/oldWALs
2024-12-09T17:10:27,955 INFO  [RS:0;80c69eb3c456:43437 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C43437%2C1733764138692:(num 1733764227289)
2024-12-09T17:10:27,955 DEBUG [RS:0;80c69eb3c456:43437 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:27,955 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:10:27,955 INFO  [RS:0;80c69eb3c456:43437 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-09T17:10:27,955 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:10:27,955 INFO  [RS:0;80c69eb3c456:43437 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43437
2024-12-09T17:10:27,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,43437,1733764138692
2024-12-09T17:10:27,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:10:27,972 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,43437,1733764138692]
2024-12-09T17:10:27,972 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,43437,1733764138692; numProcessing=1
2024-12-09T17:10:27,981 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,43437,1733764138692 already deleted, retry=false
2024-12-09T17:10:27,981 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,43437,1733764138692 expired; onlineServers=0
2024-12-09T17:10:27,981 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,46085,1733764138548' *****
2024-12-09T17:10:27,981 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-09T17:10:27,981 DEBUG [M:0;80c69eb3c456:46085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c1caa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:10:27,981 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,46085,1733764138548
2024-12-09T17:10:27,981 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,46085,1733764138548; all regions closed.
2024-12-09T17:10:27,981 DEBUG [M:0;80c69eb3c456:46085 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:27,981 DEBUG [M:0;80c69eb3c456:46085 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-09T17:10:27,981 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-09T17:10:27,981 DEBUG [M:0;80c69eb3c456:46085 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-09T17:10:27,981 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764138967 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764138967,5,FailOnTimeoutGroup]
2024-12-09T17:10:27,981 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764138968 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764138968,5,FailOnTimeoutGroup]
2024-12-09T17:10:27,981 INFO  [M:0;80c69eb3c456:46085 {}] hbase.ChoreService(370): Chore service for: master/80c69eb3c456:0 had [] on shutdown
2024-12-09T17:10:27,982 DEBUG [M:0;80c69eb3c456:46085 {}] master.HMaster(1733): Stopping service threads
2024-12-09T17:10:27,982 INFO  [M:0;80c69eb3c456:46085 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-09T17:10:27,982 INFO  [M:0;80c69eb3c456:46085 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-09T17:10:27,982 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-09T17:10:27,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-09T17:10:27,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:27,989 DEBUG [M:0;80c69eb3c456:46085 {}] zookeeper.ZKUtil(347): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-09T17:10:27,989 WARN  [M:0;80c69eb3c456:46085 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-09T17:10:27,989 INFO  [M:0;80c69eb3c456:46085 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-09T17:10:27,989 INFO  [M:0;80c69eb3c456:46085 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-09T17:10:27,989 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:10:27,989 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:27,989 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:10:27,989 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:27,989 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:10:27,989 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:27,989 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=66.45 KB heapSize=81.70 KB
2024-12-09T17:10:28,005 DEBUG [M:0;80c69eb3c456:46085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bf37020eb8cb4d8caec3aa954a4f154c is 82, key is hbase:meta,,1/info:regioninfo/1733764139643/Put/seqid=0
2024-12-09T17:10:28,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:28,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741879_1055 (size=5672)
2024-12-09T17:10:28,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741879_1055 (size=5672)
2024-12-09T17:10:28,011 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bf37020eb8cb4d8caec3aa954a4f154c
2024-12-09T17:10:28,027 DEBUG [M:0;80c69eb3c456:46085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/884ac88d900b4ecf958672be2db581fe is 751, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733764140728/Put/seqid=0
2024-12-09T17:10:28,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741880_1056 (size=7278)
2024-12-09T17:10:28,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741880_1056 (size=7278)
2024-12-09T17:10:28,031 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65.85 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/884ac88d900b4ecf958672be2db581fe
2024-12-09T17:10:28,050 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 884ac88d900b4ecf958672be2db581fe
2024-12-09T17:10:28,062 DEBUG [M:0;80c69eb3c456:46085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d2203974128146e58cfed8b1b4ca0eac is 69, key is 80c69eb3c456,43437,1733764138692/rs:state/1733764139039/Put/seqid=0
2024-12-09T17:10:28,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741881_1057 (size=5156)
2024-12-09T17:10:28,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741881_1057 (size=5156)
2024-12-09T17:10:28,066 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d2203974128146e58cfed8b1b4ca0eac
2024-12-09T17:10:28,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:28,073 INFO  [RS:0;80c69eb3c456:43437 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,43437,1733764138692; zookeeper connection closed.
2024-12-09T17:10:28,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43437-0x1000bc462a70001, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:28,073 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6231f638 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6231f638
2024-12-09T17:10:28,073 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-09T17:10:28,084 DEBUG [M:0;80c69eb3c456:46085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/21fc91ee2c364151bcc3286459231d66 is 52, key is load_balancer_on/state:d/1733764140330/Put/seqid=0
2024-12-09T17:10:28,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741882_1058 (size=5056)
2024-12-09T17:10:28,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741882_1058 (size=5056)
2024-12-09T17:10:28,088 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/21fc91ee2c364151bcc3286459231d66
2024-12-09T17:10:28,093 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bf37020eb8cb4d8caec3aa954a4f154c as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bf37020eb8cb4d8caec3aa954a4f154c
2024-12-09T17:10:28,098 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bf37020eb8cb4d8caec3aa954a4f154c, entries=8, sequenceid=164, filesize=5.5 K
2024-12-09T17:10:28,099 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/884ac88d900b4ecf958672be2db581fe as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/884ac88d900b4ecf958672be2db581fe
2024-12-09T17:10:28,103 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 884ac88d900b4ecf958672be2db581fe
2024-12-09T17:10:28,103 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/884ac88d900b4ecf958672be2db581fe, entries=18, sequenceid=164, filesize=7.1 K
2024-12-09T17:10:28,104 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d2203974128146e58cfed8b1b4ca0eac as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d2203974128146e58cfed8b1b4ca0eac
2024-12-09T17:10:28,108 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d2203974128146e58cfed8b1b4ca0eac, entries=1, sequenceid=164, filesize=5.0 K
2024-12-09T17:10:28,109 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/21fc91ee2c364151bcc3286459231d66 as hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/21fc91ee2c364151bcc3286459231d66
2024-12-09T17:10:28,112 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35833/user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/21fc91ee2c364151bcc3286459231d66, entries=1, sequenceid=164, filesize=4.9 K
2024-12-09T17:10:28,113 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(3040): Finished flush of dataSize ~66.45 KB/68043, heapSize ~81.63 KB/83592, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=164, compaction requested=false
2024-12-09T17:10:28,114 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:28,114 DEBUG [M:0;80c69eb3c456:46085 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:10:28,114 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/575118dc-d907-62dc-c9ba-33d4c832ca7b/MasterData/WALs/80c69eb3c456,46085,1733764138548
2024-12-09T17:10:28,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45139 is added to blk_1073741830_1006 (size=79272)
2024-12-09T17:10:28,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44491 is added to blk_1073741830_1006 (size=79272)
2024-12-09T17:10:28,116 INFO  [M:0;80c69eb3c456:46085 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-09T17:10:28,116 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:10:28,116 INFO  [M:0;80c69eb3c456:46085 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46085
2024-12-09T17:10:28,122 DEBUG [M:0;80c69eb3c456:46085 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/80c69eb3c456,46085,1733764138548 already deleted, retry=false
2024-12-09T17:10:28,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:28,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46085-0x1000bc462a70000, quorum=127.0.0.1:49603, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:28,231 INFO  [M:0;80c69eb3c456:46085 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,46085,1733764138548; zookeeper connection closed.
2024-12-09T17:10:28,236 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@52c967b0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:10:28,237 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3dfee23a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:10:28,237 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:10:28,237 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a4f2efe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:10:28,238 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54b978f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir/,STOPPED}
2024-12-09T17:10:28,240 WARN  [BP-1569817708-172.17.0.2-1733764136840 heartbeating to localhost/127.0.0.1:35833 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:10:28,240 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:10:28,240 WARN  [BP-1569817708-172.17.0.2-1733764136840 heartbeating to localhost/127.0.0.1:35833 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1569817708-172.17.0.2-1733764136840 (Datanode Uuid 48439287-f2c8-45d9-8aca-a98463b0e34f) service to localhost/127.0.0.1:35833
2024-12-09T17:10:28,240 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:10:28,241 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data3/current/BP-1569817708-172.17.0.2-1733764136840 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:28,242 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data4/current/BP-1569817708-172.17.0.2-1733764136840 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:28,242 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:10:28,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77ee6c4c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:10:28,244 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b54bf6a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:10:28,244 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:10:28,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7aa85543{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:10:28,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e8f61f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir/,STOPPED}
2024-12-09T17:10:28,245 WARN  [BP-1569817708-172.17.0.2-1733764136840 heartbeating to localhost/127.0.0.1:35833 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:10:28,245 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:10:28,245 WARN  [BP-1569817708-172.17.0.2-1733764136840 heartbeating to localhost/127.0.0.1:35833 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1569817708-172.17.0.2-1733764136840 (Datanode Uuid 4251a0ef-e81d-4f68-9d1d-43dcaa50ffd1) service to localhost/127.0.0.1:35833
2024-12-09T17:10:28,245 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:10:28,246 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data1/current/BP-1569817708-172.17.0.2-1733764136840 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:28,246 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/cluster_19f22f68-a132-5b44-d839-7663499d30a6/dfs/data/data2/current/BP-1569817708-172.17.0.2-1733764136840 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:28,246 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:10:28,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c76d837{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:10:28,251 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a029663{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:10:28,251 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:10:28,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73d21e1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:10:28,252 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e41ff6b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir/,STOPPED}
2024-12-09T17:10:28,258 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-09T17:10:28,286 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-09T17:10:28,292 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=124 (was 110) - Thread LEAK? -, OpenFileDescriptor=490 (was 464) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=76 (was 25) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4921 (was 4859) - AvailableMemoryMB LEAK? -
2024-12-09T17:10:28,297 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=124, OpenFileDescriptor=490, MaxFileDescriptor=1048576, SystemLoadAverage=76, ProcessCount=11, AvailableMemoryMB=4921
2024-12-09T17:10:28,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-09T17:10:28,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.log.dir so I do NOT create it in target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615
2024-12-09T17:10:28,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ee035c13-bd6b-a1dc-0f64-68ea9f60dc87/hadoop.tmp.dir so I do NOT create it in target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615
2024-12-09T17:10:28,297 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5, deleteOnExit=true
2024-12-09T17:10:28,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/test.cache.data in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.tmp.dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-09T17:10:28,298 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:10:28,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/nfs.dump.dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/java.io.tmpdir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-09T17:10:28,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-09T17:10:28,310 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:10:28,580 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:10:28,583 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:10:28,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:10:28,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:10:28,587 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-09T17:10:28,588 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:10:28,588 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f803271{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:10:28,588 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@433b5517{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:10:28,681 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@660987f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/java.io.tmpdir/jetty-localhost-33931-hadoop-hdfs-3_4_1-tests_jar-_-any-7734873106512782249/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:10:28,681 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6b7ef605{HTTP/1.1, (http/1.1)}{localhost:33931}
2024-12-09T17:10:28,681 INFO  [Time-limited test {}] server.Server(415): Started @383101ms
2024-12-09T17:10:28,692 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-09T17:10:28,855 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:10:28,857 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:10:28,858 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:10:28,858 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:10:28,858 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:10:28,858 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10c76a12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:10:28,859 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62d5846b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:10:28,946 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7a2639eb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/java.io.tmpdir/jetty-localhost-34853-hadoop-hdfs-3_4_1-tests_jar-_-any-1204435974873142310/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:10:28,946 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3173000{HTTP/1.1, (http/1.1)}{localhost:34853}
2024-12-09T17:10:28,946 INFO  [Time-limited test {}] server.Server(415): Started @383366ms
2024-12-09T17:10:28,947 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:10:28,971 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-09T17:10:28,973 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-09T17:10:28,974 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-09T17:10:28,974 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-09T17:10:28,974 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-09T17:10:28,974 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29292f37{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir/,AVAILABLE}
2024-12-09T17:10:28,975 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@66f07508{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-09T17:10:29,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:29,064 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20d0cd7d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/java.io.tmpdir/jetty-localhost-45703-hadoop-hdfs-3_4_1-tests_jar-_-any-4040019594183486693/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:10:29,065 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34dccb8d{HTTP/1.1, (http/1.1)}{localhost:45703}
2024-12-09T17:10:29,065 INFO  [Time-limited test {}] server.Server(415): Started @383484ms
2024-12-09T17:10:29,065 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-09T17:10:29,083 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:10:29,791 WARN  [Thread-2259 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data1/current/BP-1167028664-172.17.0.2-1733764228328/current, will proceed with Du for space computation calculation, 
2024-12-09T17:10:29,791 WARN  [Thread-2260 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data2/current/BP-1167028664-172.17.0.2-1733764228328/current, will proceed with Du for space computation calculation, 
2024-12-09T17:10:29,807 WARN  [Thread-2223 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:10:29,810 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x92cd5a3e4f964675 with lease ID 0xaec6041444d216ff: Processing first storage report for DS-c6dc1198-2076-492e-82b8-e84aa92d729b from datanode DatanodeRegistration(127.0.0.1:41225, datanodeUuid=204876b4-4780-416c-a005-3bda3c16b650, infoPort=46405, infoSecurePort=0, ipcPort=43845, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328)
2024-12-09T17:10:29,810 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x92cd5a3e4f964675 with lease ID 0xaec6041444d216ff: from storage DS-c6dc1198-2076-492e-82b8-e84aa92d729b node DatanodeRegistration(127.0.0.1:41225, datanodeUuid=204876b4-4780-416c-a005-3bda3c16b650, infoPort=46405, infoSecurePort=0, ipcPort=43845, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:10:29,810 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x92cd5a3e4f964675 with lease ID 0xaec6041444d216ff: Processing first storage report for DS-5fff7692-84f4-4fb7-9eeb-b734cc4f2561 from datanode DatanodeRegistration(127.0.0.1:41225, datanodeUuid=204876b4-4780-416c-a005-3bda3c16b650, infoPort=46405, infoSecurePort=0, ipcPort=43845, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328)
2024-12-09T17:10:29,810 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x92cd5a3e4f964675 with lease ID 0xaec6041444d216ff: from storage DS-5fff7692-84f4-4fb7-9eeb-b734cc4f2561 node DatanodeRegistration(127.0.0.1:41225, datanodeUuid=204876b4-4780-416c-a005-3bda3c16b650, infoPort=46405, infoSecurePort=0, ipcPort=43845, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:10:29,929 WARN  [Thread-2270 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data3/current/BP-1167028664-172.17.0.2-1733764228328/current, will proceed with Du for space computation calculation, 
2024-12-09T17:10:29,929 WARN  [Thread-2271 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data4/current/BP-1167028664-172.17.0.2-1733764228328/current, will proceed with Du for space computation calculation, 
2024-12-09T17:10:29,946 WARN  [Thread-2246 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-09T17:10:29,948 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x821b0320f461e80 with lease ID 0xaec6041444d21700: Processing first storage report for DS-169795ed-faf4-4ed8-94b6-cc3d5185fbce from datanode DatanodeRegistration(127.0.0.1:44909, datanodeUuid=d6833f45-b589-4019-ad2b-2988cf96c51f, infoPort=42353, infoSecurePort=0, ipcPort=42399, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328)
2024-12-09T17:10:29,948 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x821b0320f461e80 with lease ID 0xaec6041444d21700: from storage DS-169795ed-faf4-4ed8-94b6-cc3d5185fbce node DatanodeRegistration(127.0.0.1:44909, datanodeUuid=d6833f45-b589-4019-ad2b-2988cf96c51f, infoPort=42353, infoSecurePort=0, ipcPort=42399, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:10:29,948 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x821b0320f461e80 with lease ID 0xaec6041444d21700: Processing first storage report for DS-cb65f1ca-0c55-41d6-8175-c7b531e30099 from datanode DatanodeRegistration(127.0.0.1:44909, datanodeUuid=d6833f45-b589-4019-ad2b-2988cf96c51f, infoPort=42353, infoSecurePort=0, ipcPort=42399, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328)
2024-12-09T17:10:29,948 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x821b0320f461e80 with lease ID 0xaec6041444d21700: from storage DS-cb65f1ca-0c55-41d6-8175-c7b531e30099 node DatanodeRegistration(127.0.0.1:44909, datanodeUuid=d6833f45-b589-4019-ad2b-2988cf96c51f, infoPort=42353, infoSecurePort=0, ipcPort=42399, storageInfo=lv=-57;cid=testClusterID;nsid=87467683;c=1733764228328), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-09T17:10:29,992 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615
2024-12-09T17:10:30,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:30,017 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/zookeeper_0, clientPort=61301, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-09T17:10:30,018 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=61301
2024-12-09T17:10:30,019 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,023 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:10:30,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741825_1001 (size=7)
2024-12-09T17:10:30,032 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6 with version=8
2024-12-09T17:10:30,032 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:44739/user/jenkins/test-data/5806c747-a32c-e830-9bf6-c9f2573f7476/hbase-staging
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:10:30,034 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:10:30,035 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:35337
2024-12-09T17:10:30,035 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,036 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,038 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:35337 connecting to ZooKeeper ensemble=127.0.0.1:61301
2024-12-09T17:10:30,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353370x0, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:10:30,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35337-0x1000bc5c7fe0000 connected
2024-12-09T17:10:30,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:10:30,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:10:30,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:10:30,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35337
2024-12-09T17:10:30,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35337
2024-12-09T17:10:30,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35337
2024-12-09T17:10:30,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35337
2024-12-09T17:10:30,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35337
2024-12-09T17:10:30,179 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6, hbase.cluster.distributed=false
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/80c69eb3c456:0 server-side Connection retries=45
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-09T17:10:30,200 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-09T17:10:30,201 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44385
2024-12-09T17:10:30,201 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-09T17:10:30,203 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-09T17:10:30,204 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,206 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,209 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:44385 connecting to ZooKeeper ensemble=127.0.0.1:61301
2024-12-09T17:10:30,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443850x0, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-09T17:10:30,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:443850x0, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:10:30,216 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44385-0x1000bc5c7fe0001 connected
2024-12-09T17:10:30,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:10:30,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-09T17:10:30,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44385
2024-12-09T17:10:30,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44385
2024-12-09T17:10:30,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44385
2024-12-09T17:10:30,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44385
2024-12-09T17:10:30,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44385
2024-12-09T17:10:30,221 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,230 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80c69eb3c456:35337
2024-12-09T17:10:30,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:10:30,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:10:30,259 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:10:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-09T17:10:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,310 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:10:30,310 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80c69eb3c456,35337,1733764230033 from backup master directory
2024-12-09T17:10:30,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-09T17:10:30,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:10:30,360 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:10:30,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-09T17:10:30,360 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:10:30,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741826_1002 (size=42)
2024-12-09T17:10:30,374 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/hbase.id with ID: 328bbd3a-1d45-4098-b896-10034c73ab2b
2024-12-09T17:10:30,386 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:30,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:10:30,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741827_1003 (size=196)
2024-12-09T17:10:30,424 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-09T17:10:30,425 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-09T17:10:30,425 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:10:30,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:10:30,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741828_1004 (size=1189)
2024-12-09T17:10:30,436 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store
2024-12-09T17:10:30,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:10:30,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741829_1005 (size=34)
2024-12-09T17:10:30,443 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:10:30,444 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:10:30,444 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:30,444 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:30,444 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:10:30,444 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:30,444 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:30,444 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:10:30,444 WARN  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/.initializing
2024-12-09T17:10:30,444 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/WALs/80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,446 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C35337%2C1733764230033, suffix=, logDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/WALs/80c69eb3c456,35337,1733764230033, archiveDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/oldWALs, maxLogs=10
2024-12-09T17:10:30,446 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C35337%2C1733764230033.1733764230446
2024-12-09T17:10:30,455 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/WALs/80c69eb3c456,35337,1733764230033/80c69eb3c456%2C35337%2C1733764230033.1733764230446
2024-12-09T17:10:30,455 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46405:46405),(127.0.0.1/127.0.0.1:42353:42353)]
2024-12-09T17:10:30,456 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:10:30,456 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:10:30,456 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:10:30,456 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:10:30,457 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:10:30,458 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-09T17:10:30,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,458 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:30,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:10:30,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-09T17:10:30,459 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,460 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:10:30,460 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:10:30,461 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-09T17:10:30,461 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,461 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:10:30,461 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-09T17:10:30,462 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-09T17:10:30,462 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,462 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:10:30,463 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:10:30,463 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:10:30,465 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-09T17:10:30,465 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-09T17:10:30,467 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:10:30,467 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782530, jitterRate=-0.004961833357810974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-09T17:10:30,468 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:10:30,468 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-09T17:10:30,470 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd721e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:10:30,471 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-09T17:10:30,471 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-09T17:10:30,471 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-09T17:10:30,471 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-09T17:10:30,472 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-09T17:10:30,472 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-09T17:10:30,472 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-09T17:10:30,474 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-09T17:10:30,475 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-09T17:10:30,498 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-09T17:10:30,499 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-09T17:10:30,500 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-09T17:10:30,507 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-09T17:10:30,507 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-09T17:10:30,509 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-09T17:10:30,515 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-09T17:10:30,516 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-09T17:10:30,523 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-09T17:10:30,525 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-09T17:10:30,532 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-09T17:10:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:10:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-09T17:10:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,541 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=80c69eb3c456,35337,1733764230033, sessionid=0x1000bc5c7fe0000, setting cluster-up flag (Was=false)
2024-12-09T17:10:30,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,582 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-09T17:10:30,583 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:30,623 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-09T17:10:30,624 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80c69eb3c456,35337,1733764230033
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-09T17:10:30,627 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-09T17:10:30,627 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80c69eb3c456,35337,1733764230033
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80c69eb3c456:0, corePoolSize=5, maxPoolSize=5
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80c69eb3c456:0, corePoolSize=10, maxPoolSize=10
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,627 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:10:30,628 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,628 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733764260628
2024-12-09T17:10:30,628 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-09T17:10:30,629 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-09T17:10:30,629 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-09T17:10:30,629 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-09T17:10:30,630 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,630 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764230630,5,FailOnTimeoutGroup]
2024-12-09T17:10:30,630 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764230630,5,FailOnTimeoutGroup]
2024-12-09T17:10:30,630 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,630 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:10:30,630 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-09T17:10:30,630 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,630 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:10:30,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741831_1007 (size=1039)
2024-12-09T17:10:30,637 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-09T17:10:30,637 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6
2024-12-09T17:10:30,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:10:30,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741832_1008 (size=32)
2024-12-09T17:10:30,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:10:30,645 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:10:30,646 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:10:30,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,646 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:30,647 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:10:30,647 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:10:30,647 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,648 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:30,648 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:10:30,649 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:10:30,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:30,650 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:30,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740
2024-12-09T17:10:30,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740
2024-12-09T17:10:30,652 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:10:30,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:10:30,655 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:10:30,655 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836168, jitterRate=0.06324350833892822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:10:30,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:10:30,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:10:30,656 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:10:30,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:10:30,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:10:30,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:10:30,656 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:10:30,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:10:30,657 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-09T17:10:30,657 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-09T17:10:30,657 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-09T17:10:30,658 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-09T17:10:30,659 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-09T17:10:30,731 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80c69eb3c456:44385
2024-12-09T17:10:30,732 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1008): ClusterId : 328bbd3a-1d45-4098-b896-10034c73ab2b
2024-12-09T17:10:30,732 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-09T17:10:30,759 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-09T17:10:30,759 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-09T17:10:30,799 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-09T17:10:30,799 DEBUG [RS:0;80c69eb3c456:44385 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ca53d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:10:30,800 DEBUG [RS:0;80c69eb3c456:44385 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625f2dda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:10:30,800 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-09T17:10:30,800 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-09T17:10:30,800 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-09T17:10:30,800 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(3073): reportForDuty to master=80c69eb3c456,35337,1733764230033 with isa=80c69eb3c456/172.17.0.2:44385, startcode=1733764230199
2024-12-09T17:10:30,800 DEBUG [RS:0;80c69eb3c456:44385 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-09T17:10:30,802 INFO  [RS-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45911, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-09T17:10:30,803 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35337 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 80c69eb3c456,44385,1733764230199
2024-12-09T17:10:30,803 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35337 {}] master.ServerManager(486): Registering regionserver=80c69eb3c456,44385,1733764230199
2024-12-09T17:10:30,804 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6
2024-12-09T17:10:30,804 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35993
2024-12-09T17:10:30,804 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-09T17:10:30,809 WARN  [80c69eb3c456:35337 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-09T17:10:30,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:10:30,815 DEBUG [RS:0;80c69eb3c456:44385 {}] zookeeper.ZKUtil(111): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80c69eb3c456,44385,1733764230199
2024-12-09T17:10:30,815 WARN  [RS:0;80c69eb3c456:44385 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-09T17:10:30,815 INFO  [RS:0;80c69eb3c456:44385 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:10:30,815 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199
2024-12-09T17:10:30,818 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80c69eb3c456,44385,1733764230199]
2024-12-09T17:10:30,821 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-09T17:10:30,821 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-09T17:10:30,823 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-09T17:10:30,823 INFO  [RS:0;80c69eb3c456:44385 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-09T17:10:30,823 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,824 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-09T17:10:30,825 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80c69eb3c456:0, corePoolSize=2, maxPoolSize=2
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80c69eb3c456:0, corePoolSize=1, maxPoolSize=1
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:10:30,825 DEBUG [RS:0;80c69eb3c456:44385 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80c69eb3c456:0, corePoolSize=3, maxPoolSize=3
2024-12-09T17:10:30,826 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,826 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,826 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,826 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,826 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44385,1733764230199-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:10:30,843 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-09T17:10:30,843 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,44385,1733764230199-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:30,856 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.Replication(204): 80c69eb3c456,44385,1733764230199 started
2024-12-09T17:10:30,856 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1767): Serving as 80c69eb3c456,44385,1733764230199, RpcServer on 80c69eb3c456/172.17.0.2:44385, sessionid=0x1000bc5c7fe0001
2024-12-09T17:10:30,856 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-09T17:10:30,856 DEBUG [RS:0;80c69eb3c456:44385 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80c69eb3c456,44385,1733764230199
2024-12-09T17:10:30,856 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,44385,1733764230199'
2024-12-09T17:10:30,856 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80c69eb3c456,44385,1733764230199
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80c69eb3c456,44385,1733764230199'
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-09T17:10:30,857 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-09T17:10:30,858 DEBUG [RS:0;80c69eb3c456:44385 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-09T17:10:30,858 INFO  [RS:0;80c69eb3c456:44385 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-09T17:10:30,858 INFO  [RS:0;80c69eb3c456:44385 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-09T17:10:30,960 INFO  [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C44385%2C1733764230199, suffix=, logDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199, archiveDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs, maxLogs=32
2024-12-09T17:10:30,963 INFO  [RS:0;80c69eb3c456:44385 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C44385%2C1733764230199.1733764230963
2024-12-09T17:10:30,971 INFO  [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199/80c69eb3c456%2C44385%2C1733764230199.1733764230963
2024-12-09T17:10:30,971 DEBUG [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42353:42353),(127.0.0.1/127.0.0.1:46405:46405)]
2024-12-09T17:10:31,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:31,059 DEBUG [80c69eb3c456:35337 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-09T17:10:31,059 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80c69eb3c456,44385,1733764230199
2024-12-09T17:10:31,060 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,44385,1733764230199, state=OPENING
2024-12-09T17:10:31,073 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-09T17:10:31,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:31,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:31,112 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=80c69eb3c456,44385,1733764230199}]
2024-12-09T17:10:31,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:10:31,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:10:31,264 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,44385,1733764230199
2024-12-09T17:10:31,264 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-09T17:10:31,265 INFO  [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54828, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-09T17:10:31,268 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-09T17:10:31,268 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:10:31,270 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80c69eb3c456%2C44385%2C1733764230199.meta, suffix=.meta, logDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199, archiveDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs, maxLogs=32
2024-12-09T17:10:31,270 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 80c69eb3c456%2C44385%2C1733764230199.meta.1733764231270.meta
2024-12-09T17:10:31,279 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199/80c69eb3c456%2C44385%2C1733764230199.meta.1733764231270.meta
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46405:46405),(127.0.0.1/127.0.0.1:42353:42353)]
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-09T17:10:31,279 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:10:31,279 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-09T17:10:31,280 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-09T17:10:31,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-09T17:10:31,281 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-09T17:10:31,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:31,282 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:31,282 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-09T17:10:31,283 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-09T17:10:31,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:31,283 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:31,283 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-09T17:10:31,284 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-09T17:10:31,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:31,284 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-09T17:10:31,285 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740
2024-12-09T17:10:31,285 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740
2024-12-09T17:10:31,287 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-09T17:10:31,287 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-09T17:10:31,288 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728687, jitterRate=-0.07342691719532013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-09T17:10:31,288 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-09T17:10:31,289 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733764231263
2024-12-09T17:10:31,290 DEBUG [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-09T17:10:31,290 INFO  [RS_OPEN_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-09T17:10:31,291 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,44385,1733764230199
2024-12-09T17:10:31,291 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80c69eb3c456,44385,1733764230199, state=OPEN
2024-12-09T17:10:31,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-09T17:10:31,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:10:31,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-09T17:10:31,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:10:31,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-09T17:10:31,376 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-09T17:10:31,376 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=80c69eb3c456,44385,1733764230199 in 261 msec
2024-12-09T17:10:31,379 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-09T17:10:31,379 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 719 msec
2024-12-09T17:10:31,382 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 754 msec
2024-12-09T17:10:31,382 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733764231382, completionTime=-1
2024-12-09T17:10:31,382 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-09T17:10:31,382 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-09T17:10:31,383 DEBUG [hconnection-0xacf9ab-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:10:31,384 INFO  [RS-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54842, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:10:31,385 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-09T17:10:31,385 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733764291385
2024-12-09T17:10:31,385 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733764351385
2024-12-09T17:10:31,385 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec
2024-12-09T17:10:31,451 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,35337,1733764230033-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:31,452 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,35337,1733764230033-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:31,452 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,35337,1733764230033-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:31,452 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80c69eb3c456:35337, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:31,452 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:31,453 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-09T17:10:31,453 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-09T17:10:31,456 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-09T17:10:31,456 DEBUG [master/80c69eb3c456:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-09T17:10:31,457 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-09T17:10:31,457 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:31,459 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-09T17:10:31,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:10:31,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741835_1011 (size=358)
2024-12-09T17:10:31,467 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 32ac6cfc80e7ca47cf20d884cbc82607, NAME => 'hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6
2024-12-09T17:10:31,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:10:31,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741836_1012 (size=42)
2024-12-09T17:10:31,475 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:10:31,475 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 32ac6cfc80e7ca47cf20d884cbc82607, disabling compactions & flushes
2024-12-09T17:10:31,475 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,475 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,475 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607. after waiting 0 ms
2024-12-09T17:10:31,475 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,475 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,475 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 32ac6cfc80e7ca47cf20d884cbc82607:

2024-12-09T17:10:31,476 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-09T17:10:31,477 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733764231476"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733764231476"}]},"ts":"1733764231476"}
2024-12-09T17:10:31,478 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-09T17:10:31,479 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-09T17:10:31,479 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764231479"}]},"ts":"1733764231479"}
2024-12-09T17:10:31,480 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-09T17:10:31,560 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=32ac6cfc80e7ca47cf20d884cbc82607, ASSIGN}]
2024-12-09T17:10:31,562 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=32ac6cfc80e7ca47cf20d884cbc82607, ASSIGN
2024-12-09T17:10:31,564 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=32ac6cfc80e7ca47cf20d884cbc82607, ASSIGN; state=OFFLINE, location=80c69eb3c456,44385,1733764230199; forceNewPlan=false, retain=false
2024-12-09T17:10:31,714 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=32ac6cfc80e7ca47cf20d884cbc82607, regionState=OPENING, regionLocation=80c69eb3c456,44385,1733764230199
2024-12-09T17:10:31,717 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 32ac6cfc80e7ca47cf20d884cbc82607, server=80c69eb3c456,44385,1733764230199}]
2024-12-09T17:10:31,870 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 80c69eb3c456,44385,1733764230199
2024-12-09T17:10:31,878 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,878 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 32ac6cfc80e7ca47cf20d884cbc82607, NAME => 'hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.', STARTKEY => '', ENDKEY => ''}
2024-12-09T17:10:31,879 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:31,879 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-09T17:10:31,879 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:31,879 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:31,882 INFO  [StoreOpener-32ac6cfc80e7ca47cf20d884cbc82607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 32ac6cfc80e7ca47cf20d884cbc82607 
2024-12-09T17:10:31,883 INFO  [StoreOpener-32ac6cfc80e7ca47cf20d884cbc82607-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32ac6cfc80e7ca47cf20d884cbc82607 columnFamilyName info
2024-12-09T17:10:31,883 DEBUG [StoreOpener-32ac6cfc80e7ca47cf20d884cbc82607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-09T17:10:31,884 INFO  [StoreOpener-32ac6cfc80e7ca47cf20d884cbc82607-1 {}] regionserver.HStore(327): Store=32ac6cfc80e7ca47cf20d884cbc82607/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-09T17:10:31,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:31,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:31,887 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:31,889 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-09T17:10:31,889 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 32ac6cfc80e7ca47cf20d884cbc82607; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864514, jitterRate=0.09928762912750244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-09T17:10:31,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 32ac6cfc80e7ca47cf20d884cbc82607:

2024-12-09T17:10:31,890 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607., pid=6, masterSystemTime=1733764231870
2024-12-09T17:10:31,892 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,892 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:31,892 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=32ac6cfc80e7ca47cf20d884cbc82607, regionState=OPEN, openSeqNum=2, regionLocation=80c69eb3c456,44385,1733764230199
2024-12-09T17:10:31,895 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-09T17:10:31,895 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 32ac6cfc80e7ca47cf20d884cbc82607, server=80c69eb3c456,44385,1733764230199 in 177 msec
2024-12-09T17:10:31,897 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-09T17:10:31,897 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=32ac6cfc80e7ca47cf20d884cbc82607, ASSIGN in 335 msec
2024-12-09T17:10:31,897 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-09T17:10:31,897 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733764231897"}]},"ts":"1733764231897"}
2024-12-09T17:10:31,898 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-09T17:10:31,908 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-09T17:10:31,909 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 455 msec
2024-12-09T17:10:31,957 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-09T17:10:31,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:31,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:10:31,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:31,969 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-09T17:10:31,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:10:32,000 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec
2024-12-09T17:10:32,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:32,011 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-09T17:10:32,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-09T17:10:32,043 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 31 msec
2024-12-09T17:10:32,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-09T17:10:32,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.747sec
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,35337,1733764230033-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-09T17:10:32,107 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,35337,1733764230033-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-09T17:10:32,109 DEBUG [master/80c69eb3c456:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-09T17:10:32,109 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-09T17:10:32,109 INFO  [master/80c69eb3c456:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80c69eb3c456,35337,1733764230033-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-09T17:10:32,124 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x41a1c754 to 127.0.0.1:61301 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e4c3637
2024-12-09T17:10:32,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c95267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-09T17:10:32,135 DEBUG [hconnection-0x4fbd4ed4-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-09T17:10:32,136 INFO  [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54846, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-09T17:10:32,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=80c69eb3c456,35337,1733764230033
2024-12-09T17:10:32,137 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-09T17:10:32,139 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-09T17:10:32,139 INFO  [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-09T17:10:32,141 INFO  [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1, archiveDir=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs, maxLogs=32
2024-12-09T17:10:32,142 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733764232141
2024-12-09T17:10:32,146 INFO  [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1/test.com%2C8080%2C1.1733764232141
2024-12-09T17:10:32,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46405:46405),(127.0.0.1/127.0.0.1:42353:42353)]
2024-12-09T17:10:32,146 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733764232146
2024-12-09T17:10:32,151 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1/test.com%2C8080%2C1.1733764232141 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1/test.com%2C8080%2C1.1733764232146
2024-12-09T17:10:32,152 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46405:46405),(127.0.0.1/127.0.0.1:42353:42353)]
2024-12-09T17:10:32,152 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1/test.com%2C8080%2C1.1733764232141 is not closed yet, will try archiving it next time
2024-12-09T17:10:32,152 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1
2024-12-09T17:10:32,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741837_1013 (size=93)
2024-12-09T17:10:32,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741837_1013 (size=93)
2024-12-09T17:10:32,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741838_1014 (size=93)
2024-12-09T17:10:32,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741838_1014 (size=93)
2024-12-09T17:10:32,555 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/test.com,8080,1/test.com%2C8080%2C1.1733764232141 to hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs/test.com%2C8080%2C1.1733764232141
2024-12-09T17:10:32,559 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs
2024-12-09T17:10:32,559 INFO  [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733764232146)
2024-12-09T17:10:32,559 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-09T17:10:32,559 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x41a1c754 to 127.0.0.1:61301
2024-12-09T17:10:32,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:32,559 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-09T17:10:32,559 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=491342034, stopped=false
2024-12-09T17:10:32,559 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=80c69eb3c456,35337,1733764230033
2024-12-09T17:10:32,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:10:32,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-09T17:10:32,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:32,573 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-09T17:10:32,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:32,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:32,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:10:32,574 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,44385,1733764230199' *****
2024-12-09T17:10:32,574 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-09T17:10:32,574 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-09T17:10:32,574 INFO  [RS:0;80c69eb3c456:44385 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-09T17:10:32,574 INFO  [RS:0;80c69eb3c456:44385 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-09T17:10:32,575 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(3579): Received CLOSE for 32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:32,575 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-09T17:10:32,575 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,44385,1733764230199
2024-12-09T17:10:32,575 DEBUG [RS:0;80c69eb3c456:44385 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:32,575 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-09T17:10:32,575 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-09T17:10:32,575 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-09T17:10:32,575 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-09T17:10:32,576 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 32ac6cfc80e7ca47cf20d884cbc82607, disabling compactions & flushes
2024-12-09T17:10:32,576 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:32,576 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:32,576 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607. after waiting 0 ms
2024-12-09T17:10:32,576 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:32,576 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-09T17:10:32,576 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 32ac6cfc80e7ca47cf20d884cbc82607 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-09T17:10:32,576 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close
2024-12-09T17:10:32,576 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1603): Online Regions={32ac6cfc80e7ca47cf20d884cbc82607=hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607., 1588230740=hbase:meta,,1.1588230740}
2024-12-09T17:10:32,576 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 32ac6cfc80e7ca47cf20d884cbc82607
2024-12-09T17:10:32,576 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-09T17:10:32,576 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-09T17:10:32,577 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-09T17:10:32,577 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-09T17:10:32,577 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-09T17:10:32,577 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB
2024-12-09T17:10:32,591 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/.tmp/info/57aaa677a6e14a888f45c50a94024763 is 143, key is hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607./info:regioninfo/1733764231892/Put/seqid=0
2024-12-09T17:10:32,595 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/.tmp/info/4ea83d45565c4e569e65724f6aa8df8a is 45, key is default/info:d/1733764231974/Put/seqid=0
2024-12-09T17:10:32,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741839_1015 (size=6595)
2024-12-09T17:10:32,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741839_1015 (size=6595)
2024-12-09T17:10:32,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741840_1016 (size=5037)
2024-12-09T17:10:32,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741840_1016 (size=5037)
2024-12-09T17:10:32,601 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/.tmp/info/4ea83d45565c4e569e65724f6aa8df8a
2024-12-09T17:10:32,606 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/.tmp/info/4ea83d45565c4e569e65724f6aa8df8a as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/info/4ea83d45565c4e569e65724f6aa8df8a
2024-12-09T17:10:32,611 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/info/4ea83d45565c4e569e65724f6aa8df8a, entries=2, sequenceid=6, filesize=4.9 K
2024-12-09T17:10:32,612 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 32ac6cfc80e7ca47cf20d884cbc82607 in 36ms, sequenceid=6, compaction requested=false
2024-12-09T17:10:32,612 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-09T17:10:32,617 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/namespace/32ac6cfc80e7ca47cf20d884cbc82607/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-09T17:10:32,618 INFO  [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:32,618 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 32ac6cfc80e7ca47cf20d884cbc82607:

2024-12-09T17:10:32,618 DEBUG [RS_CLOSE_REGION-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733764231453.32ac6cfc80e7ca47cf20d884cbc82607.
2024-12-09T17:10:32,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,774 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,774 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,774 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,776 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,776 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,776 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,777 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1629): Waiting on 1588230740
2024-12-09T17:10:32,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:32,827 INFO  [regionserver/80c69eb3c456:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:10:32,828 INFO  [regionserver/80c69eb3c456:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-09T17:10:32,828 INFO  [regionserver/80c69eb3c456:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-09T17:10:32,977 DEBUG [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1629): Waiting on 1588230740
2024-12-09T17:10:32,999 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/.tmp/info/57aaa677a6e14a888f45c50a94024763
2024-12-09T17:10:33,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34441/user/jenkins/test-data/7a39ebcd-d1f4-5427-731b-22197c18a5c3/WALs/80c69eb3c456,40321,1733763964476/80c69eb3c456%2C40321%2C1733763964476.meta.1733763965476.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-09T17:10:33,019 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/.tmp/table/b11872a88de54e9db8e5ed6832a8a039 is 51, key is hbase:namespace/table:state/1733764231897/Put/seqid=0
2024-12-09T17:10:33,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741841_1017 (size=5242)
2024-12-09T17:10:33,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741841_1017 (size=5242)
2024-12-09T17:10:33,031 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/.tmp/table/b11872a88de54e9db8e5ed6832a8a039
2024-12-09T17:10:33,036 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/.tmp/info/57aaa677a6e14a888f45c50a94024763 as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/info/57aaa677a6e14a888f45c50a94024763
2024-12-09T17:10:33,040 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/info/57aaa677a6e14a888f45c50a94024763, entries=10, sequenceid=9, filesize=6.4 K
2024-12-09T17:10:33,041 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/.tmp/table/b11872a88de54e9db8e5ed6832a8a039 as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/table/b11872a88de54e9db8e5ed6832a8a039
2024-12-09T17:10:33,044 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/table/b11872a88de54e9db8e5ed6832a8a039, entries=2, sequenceid=9, filesize=5.1 K
2024-12-09T17:10:33,045 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 468ms, sequenceid=9, compaction requested=false
2024-12-09T17:10:33,045 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-09T17:10:33,048 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1
2024-12-09T17:10:33,049 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-09T17:10:33,049 INFO  [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-09T17:10:33,049 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-09T17:10:33,049 DEBUG [RS_CLOSE_META-regionserver/80c69eb3c456:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-09T17:10:33,177 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,44385,1733764230199; all regions closed.
2024-12-09T17:10:33,178 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199
2024-12-09T17:10:33,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741834_1010 (size=2484)
2024-12-09T17:10:33,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741834_1010 (size=2484)
2024-12-09T17:10:33,182 DEBUG [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs
2024-12-09T17:10:33,182 INFO  [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C44385%2C1733764230199.meta:.meta(num 1733764231270)
2024-12-09T17:10:33,182 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/WALs/80c69eb3c456,44385,1733764230199
2024-12-09T17:10:33,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741833_1009 (size=1414)
2024-12-09T17:10:33,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741833_1009 (size=1414)
2024-12-09T17:10:33,189 DEBUG [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/oldWALs
2024-12-09T17:10:33,189 INFO  [RS:0;80c69eb3c456:44385 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 80c69eb3c456%2C44385%2C1733764230199:(num 1733764230963)
2024-12-09T17:10:33,189 DEBUG [RS:0;80c69eb3c456:44385 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:33,189 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.LeaseManager(133): Closed leases
2024-12-09T17:10:33,189 INFO  [RS:0;80c69eb3c456:44385 {}] hbase.ChoreService(370): Chore service for: regionserver/80c69eb3c456:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-09T17:10:33,189 INFO  [regionserver/80c69eb3c456:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:10:33,189 INFO  [RS:0;80c69eb3c456:44385 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44385
2024-12-09T17:10:33,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80c69eb3c456,44385,1733764230199
2024-12-09T17:10:33,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-09T17:10:33,248 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80c69eb3c456,44385,1733764230199]
2024-12-09T17:10:33,248 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 80c69eb3c456,44385,1733764230199; numProcessing=1
2024-12-09T17:10:33,265 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/80c69eb3c456,44385,1733764230199 already deleted, retry=false
2024-12-09T17:10:33,265 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 80c69eb3c456,44385,1733764230199 expired; onlineServers=0
2024-12-09T17:10:33,265 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '80c69eb3c456,35337,1733764230033' *****
2024-12-09T17:10:33,265 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-09T17:10:33,265 DEBUG [M:0;80c69eb3c456:35337 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1496ca46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80c69eb3c456/172.17.0.2:0
2024-12-09T17:10:33,265 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegionServer(1224): stopping server 80c69eb3c456,35337,1733764230033
2024-12-09T17:10:33,265 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegionServer(1250): stopping server 80c69eb3c456,35337,1733764230033; all regions closed.
2024-12-09T17:10:33,265 DEBUG [M:0;80c69eb3c456:35337 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-09T17:10:33,266 DEBUG [M:0;80c69eb3c456:35337 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-09T17:10:33,266 DEBUG [M:0;80c69eb3c456:35337 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-09T17:10:33,266 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-09T17:10:33,266 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764230630 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.small.0-1733764230630,5,FailOnTimeoutGroup]
2024-12-09T17:10:33,266 DEBUG [master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764230630 {}] cleaner.HFileCleaner(306): Exit Thread[master/80c69eb3c456:0:becomeActiveMaster-HFileCleaner.large.0-1733764230630,5,FailOnTimeoutGroup]
2024-12-09T17:10:33,266 INFO  [M:0;80c69eb3c456:35337 {}] hbase.ChoreService(370): Chore service for: master/80c69eb3c456:0 had [] on shutdown
2024-12-09T17:10:33,266 DEBUG [M:0;80c69eb3c456:35337 {}] master.HMaster(1733): Stopping service threads
2024-12-09T17:10:33,266 INFO  [M:0;80c69eb3c456:35337 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-09T17:10:33,266 INFO  [M:0;80c69eb3c456:35337 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-09T17:10:33,266 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-09T17:10:33,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-09T17:10:33,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-09T17:10:33,273 DEBUG [M:0;80c69eb3c456:35337 {}] zookeeper.ZKUtil(347): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-09T17:10:33,273 WARN  [M:0;80c69eb3c456:35337 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-09T17:10:33,273 INFO  [M:0;80c69eb3c456:35337 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-09T17:10:33,274 INFO  [M:0;80c69eb3c456:35337 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-09T17:10:33,274 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-09T17:10:33,274 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-09T17:10:33,274 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:33,274 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:33,274 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-09T17:10:33,274 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:33,274 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.32 KB heapSize=32.31 KB
2024-12-09T17:10:33,281 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-09T17:10:33,282 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,282 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,282 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,282 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,283 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,283 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,283 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,283 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,290 DEBUG [M:0;80c69eb3c456:35337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b7c95f0c9b6240b8ad24415a5e38e6c2 is 82, key is hbase:meta,,1/info:regioninfo/1733764231290/Put/seqid=0
2024-12-09T17:10:33,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741842_1018 (size=5672)
2024-12-09T17:10:33,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741842_1018 (size=5672)
2024-12-09T17:10:33,295 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b7c95f0c9b6240b8ad24415a5e38e6c2
2024-12-09T17:10:33,305 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,305 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,307 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,311 DEBUG [M:0;80c69eb3c456:35337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95362bcab3cc4ce386b7fc517eb55a10 is 696, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733764231908/Put/seqid=0
2024-12-09T17:10:33,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-09T17:10:33,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741843_1019 (size=6626)
2024-12-09T17:10:33,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741843_1019 (size=6626)
2024-12-09T17:10:33,316 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.72 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95362bcab3cc4ce386b7fc517eb55a10
2024-12-09T17:10:33,331 DEBUG [M:0;80c69eb3c456:35337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/750d315c806c45d4b57ec3be0d0809e8 is 69, key is 80c69eb3c456,44385,1733764230199/rs:state/1733764230803/Put/seqid=0
2024-12-09T17:10:33,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741844_1020 (size=5156)
2024-12-09T17:10:33,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741844_1020 (size=5156)
2024-12-09T17:10:33,335 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/750d315c806c45d4b57ec3be0d0809e8
2024-12-09T17:10:33,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:33,349 INFO  [RS:0;80c69eb3c456:44385 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,44385,1733764230199; zookeeper connection closed.
2024-12-09T17:10:33,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44385-0x1000bc5c7fe0001, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:33,349 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ae7e10c {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ae7e10c
2024-12-09T17:10:33,349 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-09T17:10:33,355 DEBUG [M:0;80c69eb3c456:35337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cab19b7ec7fb477cb30119ed8d7136c1 is 52, key is load_balancer_on/state:d/1733764232138/Put/seqid=0
2024-12-09T17:10:33,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741845_1021 (size=5056)
2024-12-09T17:10:33,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741845_1021 (size=5056)
2024-12-09T17:10:33,359 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cab19b7ec7fb477cb30119ed8d7136c1
2024-12-09T17:10:33,363 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b7c95f0c9b6240b8ad24415a5e38e6c2 as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b7c95f0c9b6240b8ad24415a5e38e6c2
2024-12-09T17:10:33,368 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b7c95f0c9b6240b8ad24415a5e38e6c2, entries=8, sequenceid=70, filesize=5.5 K
2024-12-09T17:10:33,368 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95362bcab3cc4ce386b7fc517eb55a10 as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95362bcab3cc4ce386b7fc517eb55a10
2024-12-09T17:10:33,372 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95362bcab3cc4ce386b7fc517eb55a10, entries=8, sequenceid=70, filesize=6.5 K
2024-12-09T17:10:33,373 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/750d315c806c45d4b57ec3be0d0809e8 as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/750d315c806c45d4b57ec3be0d0809e8
2024-12-09T17:10:33,376 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/750d315c806c45d4b57ec3be0d0809e8, entries=1, sequenceid=70, filesize=5.0 K
2024-12-09T17:10:33,377 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cab19b7ec7fb477cb30119ed8d7136c1 as hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cab19b7ec7fb477cb30119ed8d7136c1
2024-12-09T17:10:33,380 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35993/user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cab19b7ec7fb477cb30119ed8d7136c1, entries=1, sequenceid=70, filesize=4.9 K
2024-12-09T17:10:33,381 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.32 KB/25929, heapSize ~32.25 KB/33024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 107ms, sequenceid=70, compaction requested=false
2024-12-09T17:10:33,382 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-09T17:10:33,382 DEBUG [M:0;80c69eb3c456:35337 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-09T17:10:33,382 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/1ffbb977-d559-9101-a8d3-ec477457aec6/MasterData/WALs/80c69eb3c456,35337,1733764230033
2024-12-09T17:10:33,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44909 is added to blk_1073741830_1006 (size=31030)
2024-12-09T17:10:33,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41225 is added to blk_1073741830_1006 (size=31030)
2024-12-09T17:10:33,384 INFO  [M:0;80c69eb3c456:35337 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-09T17:10:33,384 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-09T17:10:33,384 INFO  [M:0;80c69eb3c456:35337 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:35337
2024-12-09T17:10:33,390 DEBUG [M:0;80c69eb3c456:35337 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/80c69eb3c456,35337,1733764230033 already deleted, retry=false
2024-12-09T17:10:33,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:33,499 INFO  [M:0;80c69eb3c456:35337 {}] regionserver.HRegionServer(1307): Exiting; stopping=80c69eb3c456,35337,1733764230033; zookeeper connection closed.
2024-12-09T17:10:33,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35337-0x1000bc5c7fe0000, quorum=127.0.0.1:61301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-09T17:10:33,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20d0cd7d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:10:33,501 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34dccb8d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:10:33,501 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:10:33,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@66f07508{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:10:33,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29292f37{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir/,STOPPED}
2024-12-09T17:10:33,502 WARN  [BP-1167028664-172.17.0.2-1733764228328 heartbeating to localhost/127.0.0.1:35993 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:10:33,502 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:10:33,502 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:10:33,502 WARN  [BP-1167028664-172.17.0.2-1733764228328 heartbeating to localhost/127.0.0.1:35993 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1167028664-172.17.0.2-1733764228328 (Datanode Uuid d6833f45-b589-4019-ad2b-2988cf96c51f) service to localhost/127.0.0.1:35993
2024-12-09T17:10:33,502 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data3/current/BP-1167028664-172.17.0.2-1733764228328 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:33,503 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data4/current/BP-1167028664-172.17.0.2-1733764228328 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:33,503 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:10:33,504 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7a2639eb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-09T17:10:33,504 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3173000{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:10:33,504 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:10:33,505 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62d5846b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:10:33,505 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10c76a12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir/,STOPPED}
2024-12-09T17:10:33,505 WARN  [BP-1167028664-172.17.0.2-1733764228328 heartbeating to localhost/127.0.0.1:35993 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-09T17:10:33,505 WARN  [BP-1167028664-172.17.0.2-1733764228328 heartbeating to localhost/127.0.0.1:35993 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1167028664-172.17.0.2-1733764228328 (Datanode Uuid 204876b4-4780-416c-a005-3bda3c16b650) service to localhost/127.0.0.1:35993
2024-12-09T17:10:33,505 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-09T17:10:33,506 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-09T17:10:33,506 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data1/current/BP-1167028664-172.17.0.2-1733764228328 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:33,506 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/cluster_d12a2baa-d566-05ed-d191-bdc79eb3e2e5/dfs/data/data2/current/BP-1167028664-172.17.0.2-1733764228328 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-09T17:10:33,506 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-09T17:10:33,512 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@660987f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-09T17:10:33,512 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6b7ef605{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-09T17:10:33,512 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-09T17:10:33,512 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@433b5517{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-09T17:10:33,513 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f803271{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8b940597-4fbc-539c-4ed2-6344fa43f615/hadoop.log.dir/,STOPPED}
2024-12-09T17:10:33,517 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-09T17:10:33,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-09T17:10:33,550 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=143 (was 124) - Thread LEAK? -, OpenFileDescriptor=513 (was 490) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=134 (was 76) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4682 (was 4921)