2024-12-06 01:51:07,583 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-06 01:51:07,601 main DEBUG Took 0.014481 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-06 01:51:07,601 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-06 01:51:07,601 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-06 01:51:07,603 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-06 01:51:07,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,614 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-06 01:51:07,631 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,634 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,636 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,637 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,638 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,640 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,641 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,642 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,642 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,643 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,645 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,646 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,647 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-06 01:51:07,648 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,648 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-06 01:51:07,650 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-06 01:51:07,652 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-06 01:51:07,654 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-06 01:51:07,655 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-06 01:51:07,656 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-06 01:51:07,657 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-06 01:51:07,671 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-06 01:51:07,674 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-06 01:51:07,677 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-06 01:51:07,677 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-06 01:51:07,678 main DEBUG createAppenders(={Console})
2024-12-06 01:51:07,679 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-06 01:51:07,679 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-06 01:51:07,680 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-06 01:51:07,681 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-06 01:51:07,681 main DEBUG OutputStream closed
2024-12-06 01:51:07,682 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-06 01:51:07,682 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-06 01:51:07,682 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-06 01:51:07,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-06 01:51:07,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-06 01:51:07,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-06 01:51:07,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-06 01:51:07,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-06 01:51:07,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-06 01:51:07,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-06 01:51:07,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-06 01:51:07,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-06 01:51:07,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-06 01:51:07,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-06 01:51:07,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-06 01:51:07,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-06 01:51:07,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-06 01:51:07,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-06 01:51:07,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-06 01:51:07,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-06 01:51:07,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-06 01:51:07,797 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-06 01:51:07,797 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-06 01:51:07,798 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-06 01:51:07,798 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-06T01:51:08,106 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f
2024-12-06 01:51:08,110 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-06 01:51:08,111 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-06T01:51:08,122 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-06T01:51:08,168 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=240, ProcessCount=11, AvailableMemoryMB=10314
2024-12-06T01:51:08,172 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:51:08,198 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3, deleteOnExit=true
2024-12-06T01:51:08,198 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:51:08,200 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/test.cache.data in system properties and HBase conf
2024-12-06T01:51:08,200 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:51:08,201 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:51:08,202 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:51:08,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:51:08,203 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:51:08,319 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-06T01:51:08,443 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:51:08,448 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:51:08,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:51:08,450 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:51:08,450 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:51:08,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:51:08,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:51:08,452 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:51:08,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:51:08,453 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:51:08,454 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:51:08,456 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:51:08,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:51:08,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:51:08,458 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:51:09,014 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:51:09,409 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-06T01:51:09,513 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2780ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-06T01:51:09,627 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:51:09,716 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:51:09,749 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:51:09,749 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:51:09,751 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:51:09,769 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:51:09,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75bdea07{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:51:09,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@455f3457{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:51:10,037 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f961078{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/java.io.tmpdir/jetty-localhost-34469-hadoop-hdfs-3_4_1-tests_jar-_-any-17753289723554157756/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:51:10,047 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@25dfddc5{HTTP/1.1, (http/1.1)}{localhost:34469}
2024-12-06T01:51:10,047 INFO  [Time-limited test {}] server.Server(415): Started @3315ms
2024-12-06T01:51:10,079 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:51:10,489 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:51:10,496 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:51:10,497 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:51:10,497 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:51:10,497 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:51:10,498 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@616d254c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:51:10,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@198fe7a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:51:10,635 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@32c41a8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/java.io.tmpdir/jetty-localhost-37053-hadoop-hdfs-3_4_1-tests_jar-_-any-16576477665919203420/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:51:10,636 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21c64e78{HTTP/1.1, (http/1.1)}{localhost:37053}
2024-12-06T01:51:10,636 INFO  [Time-limited test {}] server.Server(415): Started @3904ms
2024-12-06T01:51:10,714 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:51:10,859 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:51:10,866 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:51:10,871 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:51:10,871 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:51:10,872 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:51:10,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1612a852{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:51:10,874 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e06ea5e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:51:11,033 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@78be0d39{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/java.io.tmpdir/jetty-localhost-39803-hadoop-hdfs-3_4_1-tests_jar-_-any-15463765419035302813/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:51:11,034 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@617aa169{HTTP/1.1, (http/1.1)}{localhost:39803}
2024-12-06T01:51:11,034 INFO  [Time-limited test {}] server.Server(415): Started @4302ms
2024-12-06T01:51:11,037 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:51:11,275 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data1/current/BP-419675471-172.17.0.2-1733449869127/current, will proceed with Du for space computation calculation, 
2024-12-06T01:51:11,275 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data3/current/BP-419675471-172.17.0.2-1733449869127/current, will proceed with Du for space computation calculation, 
2024-12-06T01:51:11,278 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data4/current/BP-419675471-172.17.0.2-1733449869127/current, will proceed with Du for space computation calculation, 
2024-12-06T01:51:11,292 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data2/current/BP-419675471-172.17.0.2-1733449869127/current, will proceed with Du for space computation calculation, 
2024-12-06T01:51:11,389 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:51:11,400 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:51:11,517 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7925df3678b5ce7 with lease ID 0x6e1c9b395dec7a53: Processing first storage report for DS-50730d79-10d1-4a8f-96b7-16a6e3296078 from datanode DatanodeRegistration(127.0.0.1:33003, datanodeUuid=5ff9979f-cb56-4871-b5a5-93cf3c12e793, infoPort=36305, infoSecurePort=0, ipcPort=33181, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127)
2024-12-06T01:51:11,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7925df3678b5ce7 with lease ID 0x6e1c9b395dec7a53: from storage DS-50730d79-10d1-4a8f-96b7-16a6e3296078 node DatanodeRegistration(127.0.0.1:33003, datanodeUuid=5ff9979f-cb56-4871-b5a5-93cf3c12e793, infoPort=36305, infoSecurePort=0, ipcPort=33181, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-06T01:51:11,521 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7925df3678b5ce7 with lease ID 0x6e1c9b395dec7a53: Processing first storage report for DS-3b8e0d96-6ef8-4c65-a449-8dd626b0820e from datanode DatanodeRegistration(127.0.0.1:33003, datanodeUuid=5ff9979f-cb56-4871-b5a5-93cf3c12e793, infoPort=36305, infoSecurePort=0, ipcPort=33181, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127)
2024-12-06T01:51:11,521 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7925df3678b5ce7 with lease ID 0x6e1c9b395dec7a53: from storage DS-3b8e0d96-6ef8-4c65-a449-8dd626b0820e node DatanodeRegistration(127.0.0.1:33003, datanodeUuid=5ff9979f-cb56-4871-b5a5-93cf3c12e793, infoPort=36305, infoSecurePort=0, ipcPort=33181, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:51:11,521 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x141aa22e93f57d7c with lease ID 0x6e1c9b395dec7a52: Processing first storage report for DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1 from datanode DatanodeRegistration(127.0.0.1:40387, datanodeUuid=58a73119-b45f-463a-b96a-d79f1c680bcd, infoPort=41963, infoSecurePort=0, ipcPort=43021, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127)
2024-12-06T01:51:11,521 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x141aa22e93f57d7c with lease ID 0x6e1c9b395dec7a52: from storage DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1 node DatanodeRegistration(127.0.0.1:40387, datanodeUuid=58a73119-b45f-463a-b96a-d79f1c680bcd, infoPort=41963, infoSecurePort=0, ipcPort=43021, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:51:11,522 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x141aa22e93f57d7c with lease ID 0x6e1c9b395dec7a52: Processing first storage report for DS-ade27a81-7530-43d2-8c85-5beecd0d72da from datanode DatanodeRegistration(127.0.0.1:40387, datanodeUuid=58a73119-b45f-463a-b96a-d79f1c680bcd, infoPort=41963, infoSecurePort=0, ipcPort=43021, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127)
2024-12-06T01:51:11,522 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x141aa22e93f57d7c with lease ID 0x6e1c9b395dec7a52: from storage DS-ade27a81-7530-43d2-8c85-5beecd0d72da node DatanodeRegistration(127.0.0.1:40387, datanodeUuid=58a73119-b45f-463a-b96a-d79f1c680bcd, infoPort=41963, infoSecurePort=0, ipcPort=43021, storageInfo=lv=-57;cid=testClusterID;nsid=577929804;c=1733449869127), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:51:11,637 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f
2024-12-06T01:51:11,749 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/zookeeper_0, clientPort=57522, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:51:11,763 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57522
2024-12-06T01:51:11,790 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:11,794 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:12,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:51:12,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:51:12,663 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18 with version=8
2024-12-06T01:51:12,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:51:12,767 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-06T01:51:13,103 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:51:13,119 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:51:13,120 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:51:13,127 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:51:13,127 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:51:13,127 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:51:13,321 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:51:13,410 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-06T01:51:13,423 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-06T01:51:13,428 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:51:13,466 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4371 (auto-detected)
2024-12-06T01:51:13,467 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-06T01:51:13,495 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43585
2024-12-06T01:51:13,524 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43585 connecting to ZooKeeper ensemble=127.0.0.1:57522
2024-12-06T01:51:13,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435850x0, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:51:13,571 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43585-0x1006505aa4c0000 connected
2024-12-06T01:51:13,617 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:13,620 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:13,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:51:13,636 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18, hbase.cluster.distributed=false
2024-12-06T01:51:13,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:51:13,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43585
2024-12-06T01:51:13,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43585
2024-12-06T01:51:13,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43585
2024-12-06T01:51:13,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43585
2024-12-06T01:51:13,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43585
2024-12-06T01:51:13,843 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:51:13,845 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:51:13,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:51:13,846 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:51:13,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:51:13,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:51:13,850 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:51:13,853 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:51:13,856 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40307
2024-12-06T01:51:13,860 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40307 connecting to ZooKeeper ensemble=127.0.0.1:57522
2024-12-06T01:51:13,861 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:13,866 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:13,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403070x0, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:51:13,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:403070x0, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:51:13,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40307-0x1006505aa4c0001 connected
2024-12-06T01:51:13,886 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:51:13,902 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:51:13,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:51:13,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:51:13,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40307
2024-12-06T01:51:13,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40307
2024-12-06T01:51:13,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40307
2024-12-06T01:51:13,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40307
2024-12-06T01:51:13,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40307
2024-12-06T01:51:13,958 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:43585
2024-12-06T01:51:13,959 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:13,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:51:13,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:51:13,973 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:14,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:51:14,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,013 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:51:14,014 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,43585,1733449872828 from backup master directory
2024-12-06T01:51:14,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:14,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:51:14,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:51:14,019 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:51:14,020 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:14,023 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-06T01:51:14,024 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-06T01:51:14,116 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase.id] with ID: c78e2e3f-2cb0-4632-adf0-97d225a5e550
2024-12-06T01:51:14,117 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/.tmp/hbase.id
2024-12-06T01:51:14,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:51:14,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:51:14,134 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/.tmp/hbase.id]:[hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase.id]
2024-12-06T01:51:14,201 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:14,208 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:51:14,229 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms.
2024-12-06T01:51:14,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:51:14,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:51:14,271 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:51:14,272 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:51:14,279 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:51:14,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:51:14,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:51:14,345 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store
2024-12-06T01:51:14,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:51:14,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:51:14,376 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-06T01:51:14,380 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:51:14,381 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:51:14,382 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:51:14,382 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:51:14,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:51:14,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:51:14,384 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:51:14,386 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733449874381Disabling compacts and flushes for region at 1733449874381Disabling writes for close at 1733449874384 (+3 ms)Writing region close event to WAL at 1733449874384Closed at 1733449874384
2024-12-06T01:51:14,389 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/.initializing
2024-12-06T01:51:14,389 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/WALs/00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:14,416 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C43585%2C1733449872828, suffix=, logDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/WALs/00c764f7c3aa,43585,1733449872828, archiveDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/oldWALs, maxLogs=10
2024-12-06T01:51:14,426 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C43585%2C1733449872828.1733449874421
2024-12-06T01:51:14,456 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/WALs/00c764f7c3aa,43585,1733449872828/00c764f7c3aa%2C43585%2C1733449872828.1733449874421
2024-12-06T01:51:14,467 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:41963:41963)]
2024-12-06T01:51:14,469 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:51:14,469 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:51:14,474 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,475 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,514 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:51:14,553 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:51:14,559 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:14,563 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:14,564 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:51:14,568 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:51:14,569 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:14,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:51:14,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:51:14,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:51:14,574 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:14,575 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:51:14,575 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:51:14,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:51:14,578 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:14,579 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:51:14,579 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,583 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,584 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,591 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,592 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,597 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:51:14,602 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:51:14,608 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:51:14,610 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=727226, jitterRate=-0.07528455555438995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:51:14,620 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733449874488Initializing all the Stores at 1733449874491 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449874491Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449874492 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449874492Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449874492Cleaning up temporary data from old regions at 1733449874592 (+100 ms)Region opened successfully at 1733449874620 (+28 ms)
2024-12-06T01:51:14,622 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:51:14,663 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993315a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:51:14,697 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:51:14,708 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:51:14,708 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:51:14,711 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:51:14,713 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-06T01:51:14,718 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-06T01:51:14,718 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:51:14,753 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:51:14,766 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:51:14,768 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:51:14,772 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:51:14,774 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:51:14,776 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:51:14,778 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:51:14,783 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:51:14,785 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:51:14,787 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:51:14,788 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:51:14,807 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:51:14,809 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:51:14,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:51:14,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:51:14,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,817 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,43585,1733449872828, sessionid=0x1006505aa4c0000, setting cluster-up flag (Was=false)
2024-12-06T01:51:14,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,843 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:51:14,846 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:14,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:14,863 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:51:14,865 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:14,874 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:51:14,939 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(746): ClusterId : c78e2e3f-2cb0-4632-adf0-97d225a5e550
2024-12-06T01:51:14,942 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:51:14,948 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:51:14,949 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:51:14,953 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:51:14,954 DEBUG [RS:0;00c764f7c3aa:40307 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14df1a24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:51:14,961 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:51:14,971 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:40307
2024-12-06T01:51:14,971 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:51:14,973 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:51:14,973 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:51:14,973 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:51:14,977 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,43585,1733449872828 with port=40307, startcode=1733449873788
2024-12-06T01:51:14,977 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:51:14,984 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,43585,1733449872828
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:51:14,992 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:51:14,992 DEBUG [RS:0;00c764f7c3aa:40307 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:51:14,992 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:51:14,992 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:51:14,992 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:51:14,993 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:51:14,993 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:14,993 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:51:14,993 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,010 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:51:15,010 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:51:15,014 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733449905014
2024-12-06T01:51:15,016 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:51:15,017 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,017 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:51:15,018 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:51:15,022 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:51:15,023 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:51:15,023 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:51:15,023 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:51:15,031 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,036 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:51:15,037 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:51:15,038 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:51:15,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:51:15,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:51:15,044 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:51:15,044 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18
2024-12-06T01:51:15,046 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:51:15,046 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:51:15,049 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449875048,5,FailOnTimeoutGroup]
2024-12-06T01:51:15,052 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449875049,5,FailOnTimeoutGroup]
2024-12-06T01:51:15,052 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,052 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:51:15,054 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,054 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:51:15,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:51:15,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:51:15,084 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:51:15,088 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:51:15,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,089 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,089 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:51:15,094 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:51:15,095 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:51:15,095 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,097 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,097 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:51:15,100 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:51:15,100 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,101 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,102 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:51:15,102 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43585 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,105 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:51:15,105 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43585 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,106 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:51:15,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740
2024-12-06T01:51:15,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740
2024-12-06T01:51:15,115 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:51:15,115 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:51:15,116 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:51:15,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:51:15,123 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18
2024-12-06T01:51:15,123 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36221
2024-12-06T01:51:15,123 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:51:15,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:51:15,131 DEBUG [RS:0;00c764f7c3aa:40307 {}] zookeeper.ZKUtil(111): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,131 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:51:15,131 WARN  [RS:0;00c764f7c3aa:40307 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:51:15,131 INFO  [RS:0;00c764f7c3aa:40307 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:51:15,132 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,133 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799051, jitterRate=0.016046181321144104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:51:15,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733449875078Initializing all the Stores at 1733449875080 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449875080Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449875084 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449875084Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449875084Cleaning up temporary data from old regions at 1733449875115 (+31 ms)Region opened successfully at 1733449875137 (+22 ms)
2024-12-06T01:51:15,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:51:15,137 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:51:15,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:51:15,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:51:15,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:51:15,139 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,40307,1733449873788]
2024-12-06T01:51:15,146 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:51:15,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733449875137Disabling compacts and flushes for region at 1733449875137Disabling writes for close at 1733449875138 (+1 ms)Writing region close event to WAL at 1733449875146 (+8 ms)Closed at 1733449875146
2024-12-06T01:51:15,150 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:51:15,150 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:51:15,159 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:51:15,170 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:51:15,173 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:51:15,186 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:51:15,205 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:51:15,212 INFO  [RS:0;00c764f7c3aa:40307 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:51:15,212 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,213 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:51:15,220 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:51:15,221 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,222 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,222 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,222 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,222 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,222 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,223 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:51:15,223 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,223 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,223 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,224 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,224 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,224 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:51:15,224 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:51:15,225 DEBUG [RS:0;00c764f7c3aa:40307 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:51:15,226 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,226 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,226 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,226 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,227 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,227 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40307,1733449873788-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:51:15,247 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:51:15,249 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40307,1733449873788-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,250 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,250 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.Replication(171): 00c764f7c3aa,40307,1733449873788 started
2024-12-06T01:51:15,270 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:15,270 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,40307,1733449873788, RpcServer on 00c764f7c3aa/172.17.0.2:40307, sessionid=0x1006505aa4c0001
2024-12-06T01:51:15,271 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:51:15,271 DEBUG [RS:0;00c764f7c3aa:40307 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,272 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,40307,1733449873788'
2024-12-06T01:51:15,272 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:51:15,273 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:51:15,274 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:51:15,274 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:51:15,274 DEBUG [RS:0;00c764f7c3aa:40307 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,274 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,40307,1733449873788'
2024-12-06T01:51:15,274 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:51:15,275 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:51:15,276 DEBUG [RS:0;00c764f7c3aa:40307 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:51:15,276 INFO  [RS:0;00c764f7c3aa:40307 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:51:15,276 INFO  [RS:0;00c764f7c3aa:40307 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:51:15,324 WARN  [00c764f7c3aa:43585 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:51:15,389 INFO  [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C40307%2C1733449873788, suffix=, logDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788, archiveDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs, maxLogs=32
2024-12-06T01:51:15,393 INFO  [RS:0;00c764f7c3aa:40307 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449875392
2024-12-06T01:51:15,412 INFO  [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449875392
2024-12-06T01:51:15,423 DEBUG [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:41963:41963)]
2024-12-06T01:51:15,577 DEBUG [00c764f7c3aa:43585 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:51:15,593 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,603 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,40307,1733449873788, state=OPENING
2024-12-06T01:51:15,611 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:51:15,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:15,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:51:15,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:51:15,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:51:15,617 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:51:15,620 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,40307,1733449873788}]
2024-12-06T01:51:15,802 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:51:15,819 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50953, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:51:15,835 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:51:15,836 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:51:15,845 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C40307%2C1733449873788.meta, suffix=.meta, logDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788, archiveDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs, maxLogs=32
2024-12-06T01:51:15,848 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.meta.1733449875848.meta
2024-12-06T01:51:15,871 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.meta.1733449875848.meta
2024-12-06T01:51:15,883 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:41963:41963)]
2024-12-06T01:51:15,889 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:51:15,891 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:51:15,896 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:51:15,902 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:51:15,909 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:51:15,911 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:51:15,912 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:51:15,912 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:51:15,924 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:51:15,927 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:51:15,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,929 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,929 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:51:15,931 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:51:15,931 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,932 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,932 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:51:15,934 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:51:15,934 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,935 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,935 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:51:15,938 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:51:15,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:15,939 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:51:15,939 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:51:15,940 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740
2024-12-06T01:51:15,943 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740
2024-12-06T01:51:15,947 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:51:15,947 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:51:15,949 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:51:15,952 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:51:15,954 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802199, jitterRate=0.020049259066581726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:51:15,954 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:51:15,956 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733449875916Writing region info on filesystem at 1733449875916Initializing all the Stores at 1733449875919 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449875919Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449875924 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449875924Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449875924Cleaning up temporary data from old regions at 1733449875947 (+23 ms)Running coprocessor post-open hooks at 1733449875954 (+7 ms)Region opened successfully at 1733449875956 (+2 ms)
2024-12-06T01:51:15,964 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733449875790
2024-12-06T01:51:15,978 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:51:15,979 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:51:15,982 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,984 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,40307,1733449873788, state=OPEN
2024-12-06T01:51:15,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:51:15,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:51:15,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:51:15,990 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:51:15,991 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:15,998 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:51:15,998 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,40307,1733449873788 in 372 msec
2024-12-06T01:51:16,005 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:51:16,005 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 841 msec
2024-12-06T01:51:16,007 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:51:16,007 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:51:16,035 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:51:16,037 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,40307,1733449873788, seqNum=-1]
2024-12-06T01:51:16,077 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:51:16,079 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:51:16,110 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1950 sec
2024-12-06T01:51:16,110 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733449876110, completionTime=-1
2024-12-06T01:51:16,113 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:51:16,114 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:51:16,148 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:51:16,148 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733449936148
2024-12-06T01:51:16,148 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733449996148
2024-12-06T01:51:16,148 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec
2024-12-06T01:51:16,152 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,43585,1733449872828-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,152 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,43585,1733449872828-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,153 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,43585,1733449872828-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,154 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:43585, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,155 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,156 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,164 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:51:16,195 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.175sec
2024-12-06T01:51:16,197 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:51:16,198 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:51:16,199 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:51:16,200 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:51:16,200 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:51:16,201 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,43585,1733449872828-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:51:16,202 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,43585,1733449872828-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:51:16,213 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:51:16,215 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:51:16,215 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,43585,1733449872828-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:51:16,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5901b87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:51:16,264 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-06T01:51:16,265 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-06T01:51:16,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,43585,-1 for getting cluster id
2024-12-06T01:51:16,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:51:16,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c78e2e3f-2cb0-4632-adf0-97d225a5e550'
2024-12-06T01:51:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:51:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c78e2e3f-2cb0-4632-adf0-97d225a5e550"

2024-12-06T01:51:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e567119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:51:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,43585,-1]
2024-12-06T01:51:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:51:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:51:16,309 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:51:16,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526567d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:51:16,314 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:51:16,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,40307,1733449873788, seqNum=-1]
2024-12-06T01:51:16,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:51:16,329 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:51:16,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:16,408 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:51:16,419 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:51:16,425 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T01:51:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,43585,1733449872828
2024-12-06T01:51:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a322e2f
2024-12-06T01:51:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T01:51:16,466 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T01:51:16,470 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T01:51:16,470 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T01:51:16,476 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:51:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-06T01:51:16,489 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T01:51:16,491 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-06T01:51:16,492 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:16,494 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T01:51:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:51:16,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741835_1011 (size=389)
2024-12-06T01:51:16,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741835_1011 (size=389)
2024-12-06T01:51:16,545 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e5253e900d84aab7ca7de6abe96751d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18
2024-12-06T01:51:16,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741836_1012 (size=72)
2024-12-06T01:51:16,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741836_1012 (size=72)
2024-12-06T01:51:16,961 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:51:16,961 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 9e5253e900d84aab7ca7de6abe96751d, disabling compactions & flushes
2024-12-06T01:51:16,961 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:16,961 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:16,961 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d. after waiting 0 ms
2024-12-06T01:51:16,961 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:16,961 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:16,961 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e5253e900d84aab7ca7de6abe96751d:
Waiting for close lock at 1733449876961Disabling compacts and flushes for region at 1733449876961Disabling writes for close at 1733449876961Writing region close event to WAL at 1733449876961Closed at 1733449876961
2024-12-06T01:51:16,964 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T01:51:16,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733449876964"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733449876964"}]},"ts":"1733449876964"}
2024-12-06T01:51:16,979 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T01:51:16,982 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T01:51:16,986 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449876982"}]},"ts":"1733449876982"}
2024-12-06T01:51:16,992 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-06T01:51:16,994 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=9e5253e900d84aab7ca7de6abe96751d, ASSIGN}]
2024-12-06T01:51:16,998 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=9e5253e900d84aab7ca7de6abe96751d, ASSIGN
2024-12-06T01:51:17,000 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=9e5253e900d84aab7ca7de6abe96751d, ASSIGN; state=OFFLINE, location=00c764f7c3aa,40307,1733449873788; forceNewPlan=false, retain=false
2024-12-06T01:51:17,152 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e5253e900d84aab7ca7de6abe96751d, regionState=OPENING, regionLocation=00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:17,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=9e5253e900d84aab7ca7de6abe96751d, ASSIGN because future has completed
2024-12-06T01:51:17,161 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e5253e900d84aab7ca7de6abe96751d, server=00c764f7c3aa,40307,1733449873788}]
2024-12-06T01:51:17,324 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:17,324 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e5253e900d84aab7ca7de6abe96751d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:51:17,325 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,325 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:51:17,325 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,325 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,328 INFO  [StoreOpener-9e5253e900d84aab7ca7de6abe96751d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9e5253e900d84aab7ca7de6abe96751d 
2024-12-06T01:51:17,330 INFO  [StoreOpener-9e5253e900d84aab7ca7de6abe96751d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e5253e900d84aab7ca7de6abe96751d columnFamilyName info
2024-12-06T01:51:17,330 DEBUG [StoreOpener-9e5253e900d84aab7ca7de6abe96751d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:51:17,332 INFO  [StoreOpener-9e5253e900d84aab7ca7de6abe96751d-1 {}] regionserver.HStore(327): Store=9e5253e900d84aab7ca7de6abe96751d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:51:17,332 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,334 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,334 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,335 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,335 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,339 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,343 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:51:17,344 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9e5253e900d84aab7ca7de6abe96751d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702262, jitterRate=-0.10702843964099884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:51:17,345 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:17,345 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9e5253e900d84aab7ca7de6abe96751d:
Running coprocessor pre-open hook at 1733449877325Writing region info on filesystem at 1733449877325Initializing all the Stores at 1733449877327 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449877327Cleaning up temporary data from old regions at 1733449877336 (+9 ms)Running coprocessor post-open hooks at 1733449877345 (+9 ms)Region opened successfully at 1733449877345
2024-12-06T01:51:17,348 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d., pid=6, masterSystemTime=1733449877316
2024-12-06T01:51:17,353 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:17,353 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:17,354 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e5253e900d84aab7ca7de6abe96751d, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,40307,1733449873788
2024-12-06T01:51:17,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e5253e900d84aab7ca7de6abe96751d, server=00c764f7c3aa,40307,1733449873788 because future has completed
2024-12-06T01:51:17,366 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T01:51:17,366 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e5253e900d84aab7ca7de6abe96751d, server=00c764f7c3aa,40307,1733449873788 in 200 msec
2024-12-06T01:51:17,370 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T01:51:17,370 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=9e5253e900d84aab7ca7de6abe96751d, ASSIGN in 372 msec
2024-12-06T01:51:17,372 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T01:51:17,372 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449877372"}]},"ts":"1733449877372"}
2024-12-06T01:51:17,376 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-06T01:51:17,377 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T01:51:17,381 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 898 msec
2024-12-06T01:51:21,420 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-06T01:51:21,471 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T01:51:21,474 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-06T01:51:23,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:51:23,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T01:51:23,407 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-06T01:51:23,407 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-06T01:51:23,409 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:51:23,409 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T01:51:23,409 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T01:51:23,409 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-06T01:51:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43585 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:51:26,550 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-06T01:51:26,553 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-06T01:51:26,561 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-06T01:51:26,562 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:51:26,563 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449886563
2024-12-06T01:51:26,573 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:26,573 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:26,573 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:26,573 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:26,574 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:26,574 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449875392 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449886563
2024-12-06T01:51:26,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:41963:41963)]
2024-12-06T01:51:26,576 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449875392 is not closed yet, will try archiving it next time
2024-12-06T01:51:26,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741833_1009 (size=451)
2024-12-06T01:51:26,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741833_1009 (size=451)
2024-12-06T01:51:26,580 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449875392 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449875392
2024-12-06T01:51:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d., hostname=00c764f7c3aa,40307,1733449873788, seqNum=2]
2024-12-06T01:51:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40307 {}] regionserver.HRegion(8855): Flush requested on 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:38,627 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e5253e900d84aab7ca7de6abe96751d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:51:38,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/57ebdeb19e7b4f1ebf426873352193ff is 1080, key is row0001/info:/1733449886589/Put/seqid=0
2024-12-06T01:51:38,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741838_1014 (size=12509)
2024-12-06T01:51:38,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741838_1014 (size=12509)
2024-12-06T01:51:38,704 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/57ebdeb19e7b4f1ebf426873352193ff
2024-12-06T01:51:38,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/57ebdeb19e7b4f1ebf426873352193ff as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff
2024-12-06T01:51:38,765 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff, entries=7, sequenceid=11, filesize=12.2 K
2024-12-06T01:51:38,774 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 9e5253e900d84aab7ca7de6abe96751d in 146ms, sequenceid=11, compaction requested=false
2024-12-06T01:51:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e5253e900d84aab7ca7de6abe96751d:

2024-12-06T01:51:41,640 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:51:46,637 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449906637
2024-12-06T01:51:46,847 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:51:46,848 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:46,848 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:46,848 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:46,848 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:46,848 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:46,849 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449886563 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449906637
2024-12-06T01:51:46,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741837_1013 (size=12399)
2024-12-06T01:51:46,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741837_1013 (size=12399)
2024-12-06T01:51:46,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41963:41963),(127.0.0.1/127.0.0.1:36305:36305)]
2024-12-06T01:51:47,060 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:49,265 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:51,470 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:53,674 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40307 {}] regionserver.HRegion(8855): Flush requested on 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:51:53,675 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e5253e900d84aab7ca7de6abe96751d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:51:53,876 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:53,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/eaf2defaf21e477ab33936bc79eb550d is 1080, key is row0008/info:/1733449900626/Put/seqid=0
2024-12-06T01:51:53,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741840_1016 (size=12509)
2024-12-06T01:51:53,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741840_1016 (size=12509)
2024-12-06T01:51:53,893 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/eaf2defaf21e477ab33936bc79eb550d
2024-12-06T01:51:53,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/eaf2defaf21e477ab33936bc79eb550d as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/eaf2defaf21e477ab33936bc79eb550d
2024-12-06T01:51:53,913 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/eaf2defaf21e477ab33936bc79eb550d, entries=7, sequenceid=21, filesize=12.2 K
2024-12-06T01:51:54,115 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:54,116 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 9e5253e900d84aab7ca7de6abe96751d in 441ms, sequenceid=21, compaction requested=false
2024-12-06T01:51:54,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e5253e900d84aab7ca7de6abe96751d:

2024-12-06T01:51:54,116 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-06T01:51:54,116 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:51:54,117 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff because midkey is the same as first or last row
2024-12-06T01:51:55,879 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:56,217 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T01:51:56,217 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T01:51:58,083 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:58,085 WARN  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:58,086 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C40307%2C1733449873788:(num 1733449906637) roll requested
2024-12-06T01:51:58,087 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449918086
2024-12-06T01:51:58,300 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 211 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:51:58,300 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:58,301 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:58,301 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:58,301 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:58,301 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:51:58,302 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449906637 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449918086
2024-12-06T01:51:58,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741839_1015 (size=7739)
2024-12-06T01:51:58,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741839_1015 (size=7739)
2024-12-06T01:51:58,308 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36305:36305),(127.0.0.1/127.0.0.1:41963:41963)]
2024-12-06T01:51:58,308 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449906637 is not closed yet, will try archiving it next time
2024-12-06T01:51:58,309 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449886563 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449886563
2024-12-06T01:52:00,288 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:52:02,325 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9e5253e900d84aab7ca7de6abe96751d, had cached 0 bytes from a total of 25018
2024-12-06T01:52:02,493 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:52:04,697 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:52:06,901 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:52:08,904 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-06T01:52:08,904 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449928904
2024-12-06T01:52:11,641 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:52:13,913 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:52:13,915 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK], DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK]]
2024-12-06T01:52:13,915 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C40307%2C1733449873788:(num 1733449928904) roll requested
2024-12-06T01:52:13,915 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:13,916 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:13,916 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:13,916 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:13,916 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:13,916 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449918086 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449928904
2024-12-06T01:52:13,917 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41963:41963),(127.0.0.1/127.0.0.1:36305:36305)]
2024-12-06T01:52:13,917 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449918086 is not closed yet, will try archiving it next time
2024-12-06T01:52:13,918 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449933918
2024-12-06T01:52:13,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741841_1017 (size=4753)
2024-12-06T01:52:13,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741841_1017 (size=4753)
2024-12-06T01:52:18,921 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:18,921 WARN  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40307 {}] regionserver.HRegion(8855): Flush requested on 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:52:18,922 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e5253e900d84aab7ca7de6abe96751d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:52:18,927 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:18,927 WARN  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:20,922 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-06T01:52:23,924 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:23,924 WARN  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:23,924 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:23,924 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:23,925 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:23,925 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:23,925 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:23,925 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449928904 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449933918
2024-12-06T01:52:23,926 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41963:41963),(127.0.0.1/127.0.0.1:36305:36305)]
2024-12-06T01:52:23,926 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449928904 is not closed yet, will try archiving it next time
2024-12-06T01:52:23,926 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C40307%2C1733449873788:(num 1733449933918) roll requested
2024-12-06T01:52:23,927 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449943927
2024-12-06T01:52:23,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741842_1018 (size=1569)
2024-12-06T01:52:23,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741842_1018 (size=1569)
2024-12-06T01:52:23,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/5f3066409fe741ae9b5ff926030fe82f is 1080, key is row0015/info:/1733449915677/Put/seqid=0
2024-12-06T01:52:23,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741844_1020 (size=12509)
2024-12-06T01:52:23,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741844_1020 (size=12509)
2024-12-06T01:52:23,939 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/5f3066409fe741ae9b5ff926030fe82f
2024-12-06T01:52:23,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/5f3066409fe741ae9b5ff926030fe82f as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/5f3066409fe741ae9b5ff926030fe82f
2024-12-06T01:52:23,959 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/5f3066409fe741ae9b5ff926030fe82f, entries=7, sequenceid=31, filesize=12.2 K
2024-12-06T01:52:28,935 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:28,935 WARN  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:28,961 INFO  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:28,961 WARN  [FSHLog-0-hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18-prefix:00c764f7c3aa,40307,1733449873788 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-c5eed1d1-55cc-4be2-a195-cfb5e0a0ebb1,DISK], DatanodeInfoWithStorage[127.0.0.1:33003,DS-50730d79-10d1-4a8f-96b7-16a6e3296078,DISK]]
2024-12-06T01:52:28,961 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,961 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 9e5253e900d84aab7ca7de6abe96751d in 10040ms, sequenceid=31, compaction requested=true
2024-12-06T01:52:28,961 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e5253e900d84aab7ca7de6abe96751d:

2024-12-06T01:52:28,962 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,962 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-06T01:52:28,962 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:52:28,962 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449933918 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449943927
2024-12-06T01:52:28,962 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff because midkey is the same as first or last row
2024-12-06T01:52:28,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741843_1019 (size=438)
2024-12-06T01:52:28,966 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41963:41963),(127.0.0.1/127.0.0.1:36305:36305)]
2024-12-06T01:52:28,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741843_1019 (size=438)
2024-12-06T01:52:28,966 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449933918 is not closed yet, will try archiving it next time
2024-12-06T01:52:28,966 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449906637 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449906637
2024-12-06T01:52:28,966 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C40307%2C1733449873788:(num 1733449943927) roll requested
2024-12-06T01:52:28,967 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449948966
2024-12-06T01:52:28,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e5253e900d84aab7ca7de6abe96751d:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:52:28,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:52:28,970 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:52:28,970 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449918086 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449918086
2024-12-06T01:52:28,973 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449928904 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449928904
2024-12-06T01:52:28,975 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:52:28,975 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449933918 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449933918
2024-12-06T01:52:28,977 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.HStore(1541): 9e5253e900d84aab7ca7de6abe96751d/info is initiating minor compaction (all files)
2024-12-06T01:52:28,978 INFO  [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e5253e900d84aab7ca7de6abe96751d/info in TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:52:28,978 INFO  [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/eaf2defaf21e477ab33936bc79eb550d, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/5f3066409fe741ae9b5ff926030fe82f] into tmpdir=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp, totalSize=36.6 K
2024-12-06T01:52:28,979 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ebdeb19e7b4f1ebf426873352193ff, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733449886589
2024-12-06T01:52:28,980 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaf2defaf21e477ab33936bc79eb550d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733449900626
2024-12-06T01:52:28,981 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f3066409fe741ae9b5ff926030fe82f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733449915677
2024-12-06T01:52:28,990 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,990 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,991 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,991 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,991 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:28,991 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449943927 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449948966
2024-12-06T01:52:28,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741845_1021 (size=93)
2024-12-06T01:52:28,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741845_1021 (size=93)
2024-12-06T01:52:29,007 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41963:41963),(127.0.0.1/127.0.0.1:36305:36305)]
2024-12-06T01:52:29,007 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449943927 is not closed yet, will try archiving it next time
2024-12-06T01:52:29,007 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C40307%2C1733449873788.1733449949007
2024-12-06T01:52:29,031 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:29,031 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:29,031 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:29,031 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:29,031 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:29,032 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449948966 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449949007
2024-12-06T01:52:29,034 INFO  [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e5253e900d84aab7ca7de6abe96751d#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:52:29,035 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/17753915878e4ff5b5140381e41a8bc7 is 1080, key is row0001/info:/1733449886589/Put/seqid=0
2024-12-06T01:52:29,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741846_1022 (size=1258)
2024-12-06T01:52:29,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741846_1022 (size=1258)
2024-12-06T01:52:29,041 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449943927 is not closed yet, will try archiving it next time
2024-12-06T01:52:29,045 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41963:41963),(127.0.0.1/127.0.0.1:36305:36305)]
2024-12-06T01:52:29,045 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449943927 is not closed yet, will try archiving it next time
2024-12-06T01:52:29,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741848_1024 (size=27710)
2024-12-06T01:52:29,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741848_1024 (size=27710)
2024-12-06T01:52:29,069 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/17753915878e4ff5b5140381e41a8bc7 as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/17753915878e4ff5b5140381e41a8bc7
2024-12-06T01:52:29,088 INFO  [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e5253e900d84aab7ca7de6abe96751d/info of 9e5253e900d84aab7ca7de6abe96751d into 17753915878e4ff5b5140381e41a8bc7(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:52:29,089 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e5253e900d84aab7ca7de6abe96751d:

2024-12-06T01:52:29,091 INFO  [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d., storeName=9e5253e900d84aab7ca7de6abe96751d/info, priority=13, startTime=1733449948964; duration=0sec
2024-12-06T01:52:29,091 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-06T01:52:29,091 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:52:29,091 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/17753915878e4ff5b5140381e41a8bc7 because midkey is the same as first or last row
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/17753915878e4ff5b5140381e41a8bc7 because midkey is the same as first or last row
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/17753915878e4ff5b5140381e41a8bc7 because midkey is the same as first or last row
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:52:29,092 DEBUG [RS:0;00c764f7c3aa:40307-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e5253e900d84aab7ca7de6abe96751d:info
2024-12-06T01:52:29,398 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/WALs/00c764f7c3aa,40307,1733449873788/00c764f7c3aa%2C40307%2C1733449873788.1733449943927 to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs/00c764f7c3aa%2C40307%2C1733449873788.1733449943927
2024-12-06T01:52:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40307 {}] regionserver.HRegion(8855): Flush requested on 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:52:41,036 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e5253e900d84aab7ca7de6abe96751d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:52:41,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/15272b75742f4e9499a259cabfe8392f is 1080, key is row0022/info:/1733449949009/Put/seqid=0
2024-12-06T01:52:41,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741849_1025 (size=12509)
2024-12-06T01:52:41,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741849_1025 (size=12509)
2024-12-06T01:52:41,055 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/15272b75742f4e9499a259cabfe8392f
2024-12-06T01:52:41,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/15272b75742f4e9499a259cabfe8392f as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/15272b75742f4e9499a259cabfe8392f
2024-12-06T01:52:41,077 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/15272b75742f4e9499a259cabfe8392f, entries=7, sequenceid=42, filesize=12.2 K
2024-12-06T01:52:41,079 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 9e5253e900d84aab7ca7de6abe96751d in 43ms, sequenceid=42, compaction requested=false
2024-12-06T01:52:41,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e5253e900d84aab7ca7de6abe96751d:

2024-12-06T01:52:41,079 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-06T01:52:41,079 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:52:41,080 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/17753915878e4ff5b5140381e41a8bc7 because midkey is the same as first or last row
2024-12-06T01:52:41,641 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:52:47,325 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9e5253e900d84aab7ca7de6abe96751d, had cached 0 bytes from a total of 40219
2024-12-06T01:52:49,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:52:49,049 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:52:49,050 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:52:49,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:49,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:49,058 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:52:49,058 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=532819799, stopped=false
2024-12-06T01:52:49,058 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,43585,1733449872828
2024-12-06T01:52:49,058 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:52:49,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:49,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:49,062 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:52:49,062 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:52:49,062 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:49,062 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:52:49,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:49,063 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,40307,1733449873788' *****
2024-12-06T01:52:49,063 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:52:49,063 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:52:49,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:49,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:49,063 INFO  [RS:0;00c764f7c3aa:40307 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:52:49,063 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:52:49,064 INFO  [RS:0;00c764f7c3aa:40307 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:52:49,064 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(3091): Received CLOSE for 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:52:49,065 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,40307,1733449873788
2024-12-06T01:52:49,065 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:52:49,065 INFO  [RS:0;00c764f7c3aa:40307 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:40307.
2024-12-06T01:52:49,065 DEBUG [RS:0;00c764f7c3aa:40307 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:52:49,065 DEBUG [RS:0;00c764f7c3aa:40307 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:49,065 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:49,066 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:52:49,066 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:52:49,066 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:52:49,066 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:52:49,066 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T01:52:49,067 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1325): Online Regions={9e5253e900d84aab7ca7de6abe96751d=TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T01:52:49,067 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9e5253e900d84aab7ca7de6abe96751d
2024-12-06T01:52:49,067 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:52:49,067 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:52:49,067 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9e5253e900d84aab7ca7de6abe96751d, disabling compactions & flushes
2024-12-06T01:52:49,067 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:52:49,067 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:52:49,067 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:52:49,067 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:52:49,067 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:52:49,068 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d. after waiting 0 ms
2024-12-06T01:52:49,068 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:52:49,068 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-06T01:52:49,068 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9e5253e900d84aab7ca7de6abe96751d 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-06T01:52:49,097 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/49ffd7d8b9934277b9669a09f44feb2c is 1080, key is row0029/info:/1733449963039/Put/seqid=0
2024-12-06T01:52:49,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741850_1026 (size=8193)
2024-12-06T01:52:49,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741850_1026 (size=8193)
2024-12-06T01:52:49,114 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/49ffd7d8b9934277b9669a09f44feb2c
2024-12-06T01:52:49,116 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/info/c1156695607f4f41b6336a2f11fafa9b is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d./info:regioninfo/1733449877354/Put/seqid=0
2024-12-06T01:52:49,132 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/.tmp/info/49ffd7d8b9934277b9669a09f44feb2c as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/49ffd7d8b9934277b9669a09f44feb2c
2024-12-06T01:52:49,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741851_1027 (size=7016)
2024-12-06T01:52:49,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741851_1027 (size=7016)
2024-12-06T01:52:49,135 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/info/c1156695607f4f41b6336a2f11fafa9b
2024-12-06T01:52:49,143 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/49ffd7d8b9934277b9669a09f44feb2c, entries=3, sequenceid=48, filesize=8.0 K
2024-12-06T01:52:49,145 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 9e5253e900d84aab7ca7de6abe96751d in 77ms, sequenceid=48, compaction requested=true
2024-12-06T01:52:49,152 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/eaf2defaf21e477ab33936bc79eb550d, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/5f3066409fe741ae9b5ff926030fe82f] to archive
2024-12-06T01:52:49,157 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T01:52:49,161 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/archive/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/57ebdeb19e7b4f1ebf426873352193ff
2024-12-06T01:52:49,164 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/eaf2defaf21e477ab33936bc79eb550d to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/archive/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/eaf2defaf21e477ab33936bc79eb550d
2024-12-06T01:52:49,174 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/ns/f504187e768340939ffce2ea1014c6af is 43, key is default/ns:d/1733449876085/Put/seqid=0
2024-12-06T01:52:49,176 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/5f3066409fe741ae9b5ff926030fe82f to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/archive/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/info/5f3066409fe741ae9b5ff926030fe82f
2024-12-06T01:52:49,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741852_1028 (size=5153)
2024-12-06T01:52:49,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741852_1028 (size=5153)
2024-12-06T01:52:49,191 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/ns/f504187e768340939ffce2ea1014c6af
2024-12-06T01:52:49,193 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=00c764f7c3aa:43585 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-06T01:52:49,199 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [57ebdeb19e7b4f1ebf426873352193ff=12509, eaf2defaf21e477ab33936bc79eb550d=12509, 5f3066409fe741ae9b5ff926030fe82f=12509]
2024-12-06T01:52:49,211 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/default/TestLogRolling-testSlowSyncLogRolling/9e5253e900d84aab7ca7de6abe96751d/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-06T01:52:49,214 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:52:49,215 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9e5253e900d84aab7ca7de6abe96751d:
Waiting for close lock at 1733449969067Running coprocessor pre-close hooks at 1733449969067Disabling compacts and flushes for region at 1733449969067Disabling writes for close at 1733449969068 (+1 ms)Obtaining lock to block concurrent updates at 1733449969068Preparing flush snapshotting stores in 9e5253e900d84aab7ca7de6abe96751d at 1733449969068Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733449969069 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d. at 1733449969071 (+2 ms)Flushing 9e5253e900d84aab7ca7de6abe96751d/info: creating writer at 1733449969071Flushing 9e5253e900d84aab7ca7de6abe96751d/info: appending metadata at 1733449969097 (+26 ms)Flushing 9e5253e900d84aab7ca7de6abe96751d/info: closing flushed file at 1733449969097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eeb695d: reopening flushed file at 1733449969130 (+33 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 9e5253e900d84aab7ca7de6abe96751d in 77ms, sequenceid=48, compaction requested=true at 1733449969145 (+15 ms)Writing region close event to WAL at 1733449969200 (+55 ms)Running coprocessor post-close hooks at 1733449969212 (+12 ms)Closed at 1733449969214 (+2 ms)
2024-12-06T01:52:49,215 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733449876468.9e5253e900d84aab7ca7de6abe96751d.
2024-12-06T01:52:49,217 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/table/5be5cfb352a14378bfa857b7fcb28d06 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733449877372/Put/seqid=0
2024-12-06T01:52:49,227 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T01:52:49,227 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T01:52:49,231 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:52:49,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741853_1029 (size=5396)
2024-12-06T01:52:49,234 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/table/5be5cfb352a14378bfa857b7fcb28d06
2024-12-06T01:52:49,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741853_1029 (size=5396)
2024-12-06T01:52:49,244 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/info/c1156695607f4f41b6336a2f11fafa9b as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/info/c1156695607f4f41b6336a2f11fafa9b
2024-12-06T01:52:49,255 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/info/c1156695607f4f41b6336a2f11fafa9b, entries=10, sequenceid=11, filesize=6.9 K
2024-12-06T01:52:49,257 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/ns/f504187e768340939ffce2ea1014c6af as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/ns/f504187e768340939ffce2ea1014c6af
2024-12-06T01:52:49,265 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/ns/f504187e768340939ffce2ea1014c6af, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T01:52:49,266 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/.tmp/table/5be5cfb352a14378bfa857b7fcb28d06 as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/table/5be5cfb352a14378bfa857b7fcb28d06
2024-12-06T01:52:49,267 DEBUG [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:52:49,275 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/table/5be5cfb352a14378bfa857b7fcb28d06, entries=2, sequenceid=11, filesize=5.3 K
2024-12-06T01:52:49,276 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 208ms, sequenceid=11, compaction requested=false
2024-12-06T01:52:49,283 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T01:52:49,284 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:52:49,284 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:52:49,286 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733449969067Running coprocessor pre-close hooks at 1733449969067Disabling compacts and flushes for region at 1733449969067Disabling writes for close at 1733449969067Obtaining lock to block concurrent updates at 1733449969068 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733449969068Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733449969069 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733449969070 (+1 ms)Flushing 1588230740/info: creating writer at 1733449969070Flushing 1588230740/info: appending metadata at 1733449969115 (+45 ms)Flushing 1588230740/info: closing flushed file at 1733449969115Flushing 1588230740/ns: creating writer at 1733449969152 (+37 ms)Flushing 1588230740/ns: appending metadata at 1733449969173 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733449969173Flushing 1588230740/table: creating writer at 1733449969200 (+27 ms)Flushing 1588230740/table: appending metadata at 1733449969217 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733449969217Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a1ee22b: reopening flushed file at 1733449969243 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a586ea7: reopening flushed file at 1733449969255 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ea29d95: reopening flushed file at 1733449969265 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 208ms, sequenceid=11, compaction requested=false at 1733449969276 (+11 ms)Writing region close event to WAL at 1733449969278 (+2 ms)Running coprocessor post-close hooks at 1733449969284 (+6 ms)Closed at 1733449969284
2024-12-06T01:52:49,286 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:52:49,468 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,40307,1733449873788; all regions closed.
2024-12-06T01:52:49,470 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,470 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,470 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,470 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,470 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741834_1010 (size=3066)
2024-12-06T01:52:49,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741834_1010 (size=3066)
2024-12-06T01:52:49,483 DEBUG [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs
2024-12-06T01:52:49,483 INFO  [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C40307%2C1733449873788.meta:.meta(num 1733449875848)
2024-12-06T01:52:49,484 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,484 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,484 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,485 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,485 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741847_1023 (size=12695)
2024-12-06T01:52:49,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741847_1023 (size=12695)
2024-12-06T01:52:49,495 DEBUG [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/oldWALs
2024-12-06T01:52:49,495 INFO  [RS:0;00c764f7c3aa:40307 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C40307%2C1733449873788:(num 1733449949007)
2024-12-06T01:52:49,495 DEBUG [RS:0;00c764f7c3aa:40307 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:49,495 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:52:49,495 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:52:49,495 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T01:52:49,495 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:52:49,495 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:52:49,496 INFO  [RS:0;00c764f7c3aa:40307 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40307
2024-12-06T01:52:49,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:52:49,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,40307,1733449873788
2024-12-06T01:52:49,501 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:52:49,503 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,40307,1733449873788]
2024-12-06T01:52:49,506 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,40307,1733449873788 already deleted, retry=false
2024-12-06T01:52:49,506 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,40307,1733449873788 expired; onlineServers=0
2024-12-06T01:52:49,506 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,43585,1733449872828' *****
2024-12-06T01:52:49,506 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:52:49,506 INFO  [M:0;00c764f7c3aa:43585 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:52:49,507 INFO  [M:0;00c764f7c3aa:43585 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:52:49,507 DEBUG [M:0;00c764f7c3aa:43585 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:52:49,507 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:52:49,507 DEBUG [M:0;00c764f7c3aa:43585 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:52:49,507 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449875048 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449875048,5,FailOnTimeoutGroup]
2024-12-06T01:52:49,507 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449875049 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449875049,5,FailOnTimeoutGroup]
2024-12-06T01:52:49,507 INFO  [M:0;00c764f7c3aa:43585 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:52:49,507 INFO  [M:0;00c764f7c3aa:43585 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:52:49,508 DEBUG [M:0;00c764f7c3aa:43585 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:52:49,508 INFO  [M:0;00c764f7c3aa:43585 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:52:49,508 INFO  [M:0;00c764f7c3aa:43585 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:52:49,508 INFO  [M:0;00c764f7c3aa:43585 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:52:49,509 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:52:49,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:52:49,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:49,510 DEBUG [M:0;00c764f7c3aa:43585 {}] zookeeper.ZKUtil(347): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:52:49,510 WARN  [M:0;00c764f7c3aa:43585 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:52:49,511 INFO  [M:0;00c764f7c3aa:43585 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/.lastflushedseqids
2024-12-06T01:52:49,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741854_1030 (size=130)
2024-12-06T01:52:49,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741854_1030 (size=130)
2024-12-06T01:52:49,538 INFO  [M:0;00c764f7c3aa:43585 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:52:49,538 INFO  [M:0;00c764f7c3aa:43585 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:52:49,539 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:52:49,539 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:49,539 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:49,539 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:52:49,539 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:49,539 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-12-06T01:52:49,559 DEBUG [M:0;00c764f7c3aa:43585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a528c177c7247b2848b68515b70f694 is 82, key is hbase:meta,,1/info:regioninfo/1733449875981/Put/seqid=0
2024-12-06T01:52:49,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741855_1031 (size=5672)
2024-12-06T01:52:49,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741855_1031 (size=5672)
2024-12-06T01:52:49,570 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a528c177c7247b2848b68515b70f694
2024-12-06T01:52:49,603 DEBUG [M:0;00c764f7c3aa:43585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1998d1078a0442ddae2c9607d307bb1f is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733449877380/Put/seqid=0
2024-12-06T01:52:49,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:49,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40307-0x1006505aa4c0001, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:49,605 INFO  [RS:0;00c764f7c3aa:40307 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:52:49,606 INFO  [RS:0;00c764f7c3aa:40307 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,40307,1733449873788; zookeeper connection closed.
2024-12-06T01:52:49,606 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fef41a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fef41a3
2024-12-06T01:52:49,607 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T01:52:49,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741856_1032 (size=6247)
2024-12-06T01:52:49,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741856_1032 (size=6247)
2024-12-06T01:52:49,612 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1998d1078a0442ddae2c9607d307bb1f
2024-12-06T01:52:49,620 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1998d1078a0442ddae2c9607d307bb1f
2024-12-06T01:52:49,644 DEBUG [M:0;00c764f7c3aa:43585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29fbd0becdbe44be8e6f24e609684c3c is 69, key is 00c764f7c3aa,40307,1733449873788/rs:state/1733449875107/Put/seqid=0
2024-12-06T01:52:49,661 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741857_1033 (size=5156)
2024-12-06T01:52:49,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741857_1033 (size=5156)
2024-12-06T01:52:49,667 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29fbd0becdbe44be8e6f24e609684c3c
2024-12-06T01:52:49,699 DEBUG [M:0;00c764f7c3aa:43585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/eb21997b50044193975f7bc350713a54 is 52, key is load_balancer_on/state:d/1733449876414/Put/seqid=0
2024-12-06T01:52:49,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741858_1034 (size=5056)
2024-12-06T01:52:49,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741858_1034 (size=5056)
2024-12-06T01:52:49,719 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/eb21997b50044193975f7bc350713a54
2024-12-06T01:52:49,727 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a528c177c7247b2848b68515b70f694 as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8a528c177c7247b2848b68515b70f694
2024-12-06T01:52:49,735 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8a528c177c7247b2848b68515b70f694, entries=8, sequenceid=59, filesize=5.5 K
2024-12-06T01:52:49,736 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1998d1078a0442ddae2c9607d307bb1f as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1998d1078a0442ddae2c9607d307bb1f
2024-12-06T01:52:49,743 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1998d1078a0442ddae2c9607d307bb1f
2024-12-06T01:52:49,743 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1998d1078a0442ddae2c9607d307bb1f, entries=6, sequenceid=59, filesize=6.1 K
2024-12-06T01:52:49,744 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29fbd0becdbe44be8e6f24e609684c3c as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29fbd0becdbe44be8e6f24e609684c3c
2024-12-06T01:52:49,751 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29fbd0becdbe44be8e6f24e609684c3c, entries=1, sequenceid=59, filesize=5.0 K
2024-12-06T01:52:49,753 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/eb21997b50044193975f7bc350713a54 as hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/eb21997b50044193975f7bc350713a54
2024-12-06T01:52:49,759 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/eb21997b50044193975f7bc350713a54, entries=1, sequenceid=59, filesize=4.9 K
2024-12-06T01:52:49,761 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 222ms, sequenceid=59, compaction requested=false
2024-12-06T01:52:49,763 INFO  [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:49,763 DEBUG [M:0;00c764f7c3aa:43585 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733449969539Disabling compacts and flushes for region at 1733449969539Disabling writes for close at 1733449969539Obtaining lock to block concurrent updates at 1733449969539Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733449969539Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733449969540 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733449969541 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733449969541Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733449969559 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733449969559Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733449969578 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733449969602 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733449969602Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733449969620 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733449969643 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733449969643Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733449969678 (+35 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733449969698 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733449969698Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4512910f: reopening flushed file at 1733449969726 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@325d5193: reopening flushed file at 1733449969735 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1125c24b: reopening flushed file at 1733449969743 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32d1332d: reopening flushed file at 1733449969752 (+9 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 222ms, sequenceid=59, compaction requested=false at 1733449969761 (+9 ms)Writing region close event to WAL at 1733449969763 (+2 ms)Closed at 1733449969763
2024-12-06T01:52:49,764 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,765 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,765 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,765 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,765 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:49,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741830_1006 (size=27973)
2024-12-06T01:52:49,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33003 is added to blk_1073741830_1006 (size=27973)
2024-12-06T01:52:49,779 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:52:49,779 INFO  [M:0;00c764f7c3aa:43585 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:52:49,780 INFO  [M:0;00c764f7c3aa:43585 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43585
2024-12-06T01:52:49,780 INFO  [M:0;00c764f7c3aa:43585 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:52:49,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:49,883 INFO  [M:0;00c764f7c3aa:43585 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:52:49,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43585-0x1006505aa4c0000, quorum=127.0.0.1:57522, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:49,889 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@78be0d39{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:49,891 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@617aa169{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:52:49,892 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:52:49,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e06ea5e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:52:49,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1612a852{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir/,STOPPED}
2024-12-06T01:52:49,895 WARN  [BP-419675471-172.17.0.2-1733449869127 heartbeating to localhost/127.0.0.1:36221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:52:49,895 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:52:49,895 WARN  [BP-419675471-172.17.0.2-1733449869127 heartbeating to localhost/127.0.0.1:36221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-419675471-172.17.0.2-1733449869127 (Datanode Uuid 58a73119-b45f-463a-b96a-d79f1c680bcd) service to localhost/127.0.0.1:36221
2024-12-06T01:52:49,895 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:52:49,897 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data3/current/BP-419675471-172.17.0.2-1733449869127 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:49,897 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data4/current/BP-419675471-172.17.0.2-1733449869127 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:49,898 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:52:49,907 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@32c41a8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:49,907 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21c64e78{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:52:49,907 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:52:49,907 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@198fe7a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:52:49,907 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@616d254c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir/,STOPPED}
2024-12-06T01:52:49,910 WARN  [BP-419675471-172.17.0.2-1733449869127 heartbeating to localhost/127.0.0.1:36221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:52:49,910 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:52:49,910 WARN  [BP-419675471-172.17.0.2-1733449869127 heartbeating to localhost/127.0.0.1:36221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-419675471-172.17.0.2-1733449869127 (Datanode Uuid 5ff9979f-cb56-4871-b5a5-93cf3c12e793) service to localhost/127.0.0.1:36221
2024-12-06T01:52:49,910 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:52:49,911 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data1/current/BP-419675471-172.17.0.2-1733449869127 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:49,911 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/cluster_71a93f68-da1a-ae53-f6da-96c1964e5da3/data/data2/current/BP-419675471-172.17.0.2-1733449869127 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:49,911 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:52:49,925 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f961078{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:52:49,926 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@25dfddc5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:52:49,926 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:52:49,927 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@455f3457{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:52:49,927 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75bdea07{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir/,STOPPED}
2024-12-06T01:52:49,940 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:52:49,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:52:50,005 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12)
Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@419be672
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/00c764f7c3aa:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36221
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:36221 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36221
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: master/00c764f7c3aa:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36221
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36221
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:36221
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:36221 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: regionserver/00c764f7c3aa:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:36221 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)
 - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 240) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9867 (was 10314)
2024-12-06T01:52:50,016 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=9865
2024-12-06T01:52:50,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:52:50,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.log.dir so I do NOT create it in target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe
2024-12-06T01:52:50,017 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c4970ac7-5488-afb7-0615-6a73d818928f/hadoop.tmp.dir so I do NOT create it in target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe
2024-12-06T01:52:50,018 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545, deleteOnExit=true
2024-12-06T01:52:50,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:52:50,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/test.cache.data in system properties and HBase conf
2024-12-06T01:52:50,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:52:50,018 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:52:50,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:52:50,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:52:50,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:52:50,019 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:52:50,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:52:50,021 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:52:50,043 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:52:50,140 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:50,152 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:52:50,157 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:52:50,157 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:52:50,157 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:52:50,158 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:50,159 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@246cb388{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:52:50,160 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73737af2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:52:50,289 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5316f6d0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/java.io.tmpdir/jetty-localhost-38877-hadoop-hdfs-3_4_1-tests_jar-_-any-10905762553296452946/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:52:50,290 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ced705f{HTTP/1.1, (http/1.1)}{localhost:38877}
2024-12-06T01:52:50,290 INFO  [Time-limited test {}] server.Server(415): Started @103558ms
2024-12-06T01:52:50,305 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:52:50,416 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:50,422 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:52:50,427 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:52:50,427 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:52:50,427 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:52:50,428 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ca07ff3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:52:50,429 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22ba3420{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:52:50,567 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a787e38{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/java.io.tmpdir/jetty-localhost-37919-hadoop-hdfs-3_4_1-tests_jar-_-any-4048664792018115138/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:50,568 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@144bd696{HTTP/1.1, (http/1.1)}{localhost:37919}
2024-12-06T01:52:50,568 INFO  [Time-limited test {}] server.Server(415): Started @103836ms
2024-12-06T01:52:50,571 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:52:50,613 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:50,617 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:52:50,619 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:52:50,619 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:52:50,619 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:52:50,620 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c9fe9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:52:50,621 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2502c8c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:52:50,691 WARN  [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data1/current/BP-1419783637-172.17.0.2-1733449970065/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:50,691 WARN  [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data2/current/BP-1419783637-172.17.0.2-1733449970065/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:50,719 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:52:50,725 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x104d93f101fcef67 with lease ID 0x7292879e5274774f: Processing first storage report for DS-bd23e3da-2a9a-4bcc-b52b-f1d60ec91bd6 from datanode DatanodeRegistration(127.0.0.1:33119, datanodeUuid=d99a1816-54c3-4d29-b749-75da1adfc1b8, infoPort=45017, infoSecurePort=0, ipcPort=42855, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065)
2024-12-06T01:52:50,725 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x104d93f101fcef67 with lease ID 0x7292879e5274774f: from storage DS-bd23e3da-2a9a-4bcc-b52b-f1d60ec91bd6 node DatanodeRegistration(127.0.0.1:33119, datanodeUuid=d99a1816-54c3-4d29-b749-75da1adfc1b8, infoPort=45017, infoSecurePort=0, ipcPort=42855, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:52:50,726 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x104d93f101fcef67 with lease ID 0x7292879e5274774f: Processing first storage report for DS-1cb5d0cd-da73-46f4-9a14-caefa9edae99 from datanode DatanodeRegistration(127.0.0.1:33119, datanodeUuid=d99a1816-54c3-4d29-b749-75da1adfc1b8, infoPort=45017, infoSecurePort=0, ipcPort=42855, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065)
2024-12-06T01:52:50,726 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x104d93f101fcef67 with lease ID 0x7292879e5274774f: from storage DS-1cb5d0cd-da73-46f4-9a14-caefa9edae99 node DatanodeRegistration(127.0.0.1:33119, datanodeUuid=d99a1816-54c3-4d29-b749-75da1adfc1b8, infoPort=45017, infoSecurePort=0, ipcPort=42855, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:52:50,757 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d2d0bcc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/java.io.tmpdir/jetty-localhost-34063-hadoop-hdfs-3_4_1-tests_jar-_-any-3163919279200992454/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:50,758 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@57c7b86a{HTTP/1.1, (http/1.1)}{localhost:34063}
2024-12-06T01:52:50,758 INFO  [Time-limited test {}] server.Server(415): Started @104025ms
2024-12-06T01:52:50,760 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:52:50,857 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data3/current/BP-1419783637-172.17.0.2-1733449970065/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:50,857 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data4/current/BP-1419783637-172.17.0.2-1733449970065/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:50,874 WARN  [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:52:50,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17a32dbcce96fdf5 with lease ID 0x7292879e52747750: Processing first storage report for DS-1b52ff31-1cf8-4c14-b118-8ac058910821 from datanode DatanodeRegistration(127.0.0.1:35601, datanodeUuid=29298d03-5e89-46c7-a60f-1eff67bd1793, infoPort=45929, infoSecurePort=0, ipcPort=37695, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065)
2024-12-06T01:52:50,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17a32dbcce96fdf5 with lease ID 0x7292879e52747750: from storage DS-1b52ff31-1cf8-4c14-b118-8ac058910821 node DatanodeRegistration(127.0.0.1:35601, datanodeUuid=29298d03-5e89-46c7-a60f-1eff67bd1793, infoPort=45929, infoSecurePort=0, ipcPort=37695, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:52:50,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17a32dbcce96fdf5 with lease ID 0x7292879e52747750: Processing first storage report for DS-e3a5990c-efac-41fd-8ddb-5fb8270a92af from datanode DatanodeRegistration(127.0.0.1:35601, datanodeUuid=29298d03-5e89-46c7-a60f-1eff67bd1793, infoPort=45929, infoSecurePort=0, ipcPort=37695, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065)
2024-12-06T01:52:50,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17a32dbcce96fdf5 with lease ID 0x7292879e52747750: from storage DS-e3a5990c-efac-41fd-8ddb-5fb8270a92af node DatanodeRegistration(127.0.0.1:35601, datanodeUuid=29298d03-5e89-46c7-a60f-1eff67bd1793, infoPort=45929, infoSecurePort=0, ipcPort=37695, storageInfo=lv=-57;cid=testClusterID;nsid=1511028273;c=1733449970065), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:52:50,896 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe
2024-12-06T01:52:50,899 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/zookeeper_0, clientPort=59806, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:52:50,900 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59806
2024-12-06T01:52:50,901 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:50,902 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:50,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:52:50,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:52:50,919 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a with version=8
2024-12-06T01:52:50,919 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:52:50,921 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:52:50,922 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:52:50,923 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42871
2024-12-06T01:52:50,924 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42871 connecting to ZooKeeper ensemble=127.0.0.1:59806
2024-12-06T01:52:50,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428710x0, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:52:50,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42871-0x10065072db90000 connected
2024-12-06T01:52:50,945 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:50,947 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:50,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:50,950 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a, hbase.cluster.distributed=false
2024-12-06T01:52:50,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:52:50,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42871
2024-12-06T01:52:50,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42871
2024-12-06T01:52:50,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42871
2024-12-06T01:52:50,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42871
2024-12-06T01:52:50,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42871
2024-12-06T01:52:50,971 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:52:50,972 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:52:50,973 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33527
2024-12-06T01:52:50,975 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33527 connecting to ZooKeeper ensemble=127.0.0.1:59806
2024-12-06T01:52:50,975 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:50,978 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:50,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335270x0, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:52:50,984 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:335270x0, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:50,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33527-0x10065072db90001 connected
2024-12-06T01:52:50,984 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:52:50,985 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:52:50,986 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:52:50,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:52:50,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33527
2024-12-06T01:52:50,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33527
2024-12-06T01:52:50,993 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33527
2024-12-06T01:52:51,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33527
2024-12-06T01:52:51,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33527
2024-12-06T01:52:51,022 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:42871
2024-12-06T01:52:51,023 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:51,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:51,028 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:52:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,031 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:52:51,031 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,42871,1733449970921 from backup master directory
2024-12-06T01:52:51,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:51,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:51,033 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:52:51,033 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,039 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/hbase.id] with ID: 31410a7e-14fd-42e1-87e7-e01315a0ef8a
2024-12-06T01:52:51,039 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/.tmp/hbase.id
2024-12-06T01:52:51,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:52:51,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:52:51,048 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/.tmp/hbase.id]:[hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/hbase.id]
2024-12-06T01:52:51,066 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:51,066 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:52:51,069 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-12-06T01:52:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:52:51,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:52:51,087 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:52:51,089 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:52:51,089 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:51,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:52:51,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:52:51,110 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store
2024-12-06T01:52:51,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:52:51,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:52:51,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:51,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:52:51,119 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:51,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:51,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:52:51,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:51,119 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:51,120 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733449971119Disabling compacts and flushes for region at 1733449971119Disabling writes for close at 1733449971119Writing region close event to WAL at 1733449971119Closed at 1733449971119
2024-12-06T01:52:51,121 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/.initializing
2024-12-06T01:52:51,121 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/WALs/00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,124 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C42871%2C1733449970921, suffix=, logDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/WALs/00c764f7c3aa,42871,1733449970921, archiveDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/oldWALs, maxLogs=10
2024-12-06T01:52:51,125 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42871%2C1733449970921.1733449971124
2024-12-06T01:52:51,131 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/WALs/00c764f7c3aa,42871,1733449970921/00c764f7c3aa%2C42871%2C1733449970921.1733449971124
2024-12-06T01:52:51,132 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45929:45929),(127.0.0.1/127.0.0.1:45017:45017)]
2024-12-06T01:52:51,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:52:51,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:51,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:51,139 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:52:51,139 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,140 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,140 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:51,142 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:52:51,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,143 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:51,143 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:51,145 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:52:51,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:51,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:51,148 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:52:51,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:51,149 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,150 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,151 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,153 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,153 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,153 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:52:51,155 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:51,158 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:52:51,158 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=729482, jitterRate=-0.07241642475128174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:52:51,160 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733449971135Initializing all the Stores at 1733449971137 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971137Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449971137Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449971137Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449971137Cleaning up temporary data from old regions at 1733449971153 (+16 ms)Region opened successfully at 1733449971160 (+7 ms)
2024-12-06T01:52:51,160 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:52:51,165 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376c05ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:52:51,166 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:52:51,166 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:52:51,166 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:52:51,166 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:52:51,167 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T01:52:51,168 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T01:52:51,168 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:52:51,174 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:52:51,175 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:52:51,177 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:52:51,177 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:52:51,178 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:52:51,179 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:52:51,180 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:52:51,182 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:52:51,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:52:51,185 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:52:51,186 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:52:51,188 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:52:51,190 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:52:51,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:51,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:51,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,192 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,42871,1733449970921, sessionid=0x10065072db90000, setting cluster-up flag (Was=false)
2024-12-06T01:52:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,204 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:52:51,206 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,216 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:52:51,218 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:51,220 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:52:51,222 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:51,222 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:52:51,223 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:52:51,223 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,42871,1733449970921
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:52:51,224 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:52:51,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733450001229
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:52:51,230 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:51,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,230 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:52:51,231 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:52:51,231 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:52:51,231 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:52:51,231 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:52:51,231 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:52:51,232 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449971231,5,FailOnTimeoutGroup]
2024-12-06T01:52:51,232 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,232 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:52:51,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449971232,5,FailOnTimeoutGroup]
2024-12-06T01:52:51,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:52:51,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:52:51,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:52:51,245 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:52:51,245 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a
2024-12-06T01:52:51,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:52:51,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:52:51,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:51,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:52:51,270 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:52:51,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,271 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,271 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:52:51,273 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:52:51,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,274 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,274 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:52:51,276 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:52:51,276 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,277 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,277 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:52:51,279 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:52:51,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,279 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:52:51,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740
2024-12-06T01:52:51,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740
2024-12-06T01:52:51,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:52:51,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:52:51,284 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:52:51,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:52:51,290 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:52:51,290 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846845, jitterRate=0.07681937515735626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:52:51,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733449971262Initializing all the Stores at 1733449971263 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971263Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971268 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449971268Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971268Cleaning up temporary data from old regions at 1733449971283 (+15 ms)Region opened successfully at 1733449971292 (+9 ms)
2024-12-06T01:52:51,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:52:51,293 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:52:51,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:52:51,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:52:51,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:52:51,294 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:52:51,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733449971293Disabling compacts and flushes for region at 1733449971293Disabling writes for close at 1733449971293Writing region close event to WAL at 1733449971294 (+1 ms)Closed at 1733449971294
2024-12-06T01:52:51,296 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:51,296 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:52:51,296 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:52:51,298 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:52:51,300 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:52:51,309 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(746): ClusterId : 31410a7e-14fd-42e1-87e7-e01315a0ef8a
2024-12-06T01:52:51,309 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:52:51,312 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:52:51,312 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:52:51,315 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:52:51,316 DEBUG [RS:0;00c764f7c3aa:33527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0a0b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:52:51,334 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:33527
2024-12-06T01:52:51,334 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:52:51,334 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:52:51,334 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:52:51,336 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,42871,1733449970921 with port=33527, startcode=1733449970971
2024-12-06T01:52:51,336 DEBUG [RS:0;00c764f7c3aa:33527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:52:51,340 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:52:51,341 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,341 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42871 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,343 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a
2024-12-06T01:52:51,343 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39051
2024-12-06T01:52:51,343 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:52:51,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:52:51,346 DEBUG [RS:0;00c764f7c3aa:33527 {}] zookeeper.ZKUtil(111): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,346 WARN  [RS:0;00c764f7c3aa:33527 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:52:51,346 INFO  [RS:0;00c764f7c3aa:33527 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:51,347 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/WALs/00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,348 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,33527,1733449970971]
2024-12-06T01:52:51,353 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:52:51,357 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:52:51,357 INFO  [RS:0;00c764f7c3aa:33527 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:52:51,357 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,358 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:52:51,359 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:52:51,359 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,359 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,359 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,359 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,359 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:51,360 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:52:51,361 DEBUG [RS:0;00c764f7c3aa:33527 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:52:51,364 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,364 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,364 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,364 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,364 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,364 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33527,1733449970971-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:52:51,389 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:52:51,389 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33527,1733449970971-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,389 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,389 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.Replication(171): 00c764f7c3aa,33527,1733449970971 started
2024-12-06T01:52:51,414 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,414 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,33527,1733449970971, RpcServer on 00c764f7c3aa/172.17.0.2:33527, sessionid=0x10065072db90001
2024-12-06T01:52:51,414 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:52:51,414 DEBUG [RS:0;00c764f7c3aa:33527 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,414 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,33527,1733449970971'
2024-12-06T01:52:51,414 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:52:51,415 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:52:51,416 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:52:51,416 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:52:51,416 DEBUG [RS:0;00c764f7c3aa:33527 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,416 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,33527,1733449970971'
2024-12-06T01:52:51,416 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:52:51,417 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:52:51,418 DEBUG [RS:0;00c764f7c3aa:33527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:52:51,418 INFO  [RS:0;00c764f7c3aa:33527 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:52:51,418 INFO  [RS:0;00c764f7c3aa:33527 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:52:51,451 WARN  [00c764f7c3aa:42871 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:52:51,522 INFO  [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C33527%2C1733449970971, suffix=, logDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/WALs/00c764f7c3aa,33527,1733449970971, archiveDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/oldWALs, maxLogs=32
2024-12-06T01:52:51,524 INFO  [RS:0;00c764f7c3aa:33527 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C33527%2C1733449970971.1733449971524
2024-12-06T01:52:51,541 INFO  [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/WALs/00c764f7c3aa,33527,1733449970971/00c764f7c3aa%2C33527%2C1733449970971.1733449971524
2024-12-06T01:52:51,552 DEBUG [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45929:45929),(127.0.0.1/127.0.0.1:45017:45017)]
2024-12-06T01:52:51,701 DEBUG [00c764f7c3aa:42871 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:52:51,702 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,705 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,33527,1733449970971, state=OPENING
2024-12-06T01:52:51,707 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:52:51,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:51,711 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:51,711 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:52:51,711 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,33527,1733449970971}]
2024-12-06T01:52:51,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:51,867 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:52:51,870 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54373, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:52:51,874 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:52:51,875 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:51,877 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C33527%2C1733449970971.meta, suffix=.meta, logDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/WALs/00c764f7c3aa,33527,1733449970971, archiveDir=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/oldWALs, maxLogs=32
2024-12-06T01:52:51,879 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C33527%2C1733449970971.meta.1733449971879.meta
2024-12-06T01:52:51,885 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/WALs/00c764f7c3aa,33527,1733449970971/00c764f7c3aa%2C33527%2C1733449970971.meta.1733449971879.meta
2024-12-06T01:52:51,886 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45017:45017),(127.0.0.1/127.0.0.1:45929:45929)]
2024-12-06T01:52:51,886 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:52:51,887 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:52:51,887 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:52:51,887 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:52:51,887 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:52:51,887 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:51,887 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:52:51,887 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:52:51,889 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:52:51,891 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:52:51,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,892 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,892 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:52:51,893 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:52:51,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:52:51,895 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:52:51,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,896 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,896 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:52:51,897 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:52:51,897 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:51,897 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:51,897 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:52:51,898 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740
2024-12-06T01:52:51,900 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740
2024-12-06T01:52:51,902 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:52:51,902 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:52:51,903 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:52:51,905 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:52:51,906 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773282, jitterRate=-0.01672123372554779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:52:51,906 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:52:51,907 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733449971888Writing region info on filesystem at 1733449971888Initializing all the Stores at 1733449971889 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971889Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971889Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449971889Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449971889Cleaning up temporary data from old regions at 1733449971902 (+13 ms)Running coprocessor post-open hooks at 1733449971906 (+4 ms)Region opened successfully at 1733449971907 (+1 ms)
2024-12-06T01:52:51,908 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733449971867
2024-12-06T01:52:51,911 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:52:51,912 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:52:51,913 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,914 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,33527,1733449970971, state=OPEN
2024-12-06T01:52:51,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:52:51,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:52:51,919 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:51,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:51,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:51,922 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:52:51,922 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,33527,1733449970971 in 208 msec
2024-12-06T01:52:51,926 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:52:51,926 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 626 msec
2024-12-06T01:52:51,927 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:51,927 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:52:51,928 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:52:51,929 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,33527,1733449970971, seqNum=-1]
2024-12-06T01:52:51,929 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:52:51,931 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53879, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:52:51,937 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 715 msec
2024-12-06T01:52:51,937 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733449971937, completionTime=-1
2024-12-06T01:52:51,937 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:52:51,937 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:52:51,939 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:52:51,939 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733450031939
2024-12-06T01:52:51,939 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733450091939
2024-12-06T01:52:51,940 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T01:52:51,940 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42871,1733449970921-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,940 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42871,1733449970921-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,940 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42871,1733449970921-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,940 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:42871, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,940 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,941 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:51,942 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.912sec
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:52:51,945 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42871,1733449970921-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:52:51,946 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42871,1733449970921-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:52:51,948 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:52:51,949 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:52:51,949 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42871,1733449970921-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:52,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367ca505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:52:52,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,42871,-1 for getting cluster id
2024-12-06T01:52:52,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:52:52,012 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31410a7e-14fd-42e1-87e7-e01315a0ef8a'
2024-12-06T01:52:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:52:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31410a7e-14fd-42e1-87e7-e01315a0ef8a"

2024-12-06T01:52:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72aebd01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:52:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,42871,-1]
2024-12-06T01:52:52,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:52:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:52,017 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:52:52,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ec83d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:52:52,019 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:52:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,33527,1733449970971, seqNum=-1]
2024-12-06T01:52:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:52:52,023 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:52:52,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:52,026 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:52,030 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:52:52,030 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:52:52,030 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:52:52,031 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:52:52,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:52,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:52,031 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:52:52,031 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:52:52,031 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1029427667, stopped=false
2024-12-06T01:52:52,031 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,42871,1733449970921
2024-12-06T01:52:52,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:52,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:52,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:52,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:52,033 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:52:52,034 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:52:52,034 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:52:52,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:52,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:52,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:52,034 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,33527,1733449970971' *****
2024-12-06T01:52:52,034 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:52:52,035 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:33527.
2024-12-06T01:52:52,035 DEBUG [RS:0;00c764f7c3aa:33527 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:52:52,035 DEBUG [RS:0;00c764f7c3aa:33527 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:52:52,035 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:52:52,036 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:52:52,036 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-06T01:52:52,036 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-06T01:52:52,036 DEBUG [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:52:52,036 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:52:52,036 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:52:52,036 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:52:52,036 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:52:52,036 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:52:52,036 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-06T01:52:52,055 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/.tmp/ns/002aab1280224de3979974891a190293 is 43, key is default/ns:d/1733449971931/Put/seqid=0
2024-12-06T01:52:52,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741835_1011 (size=5153)
2024-12-06T01:52:52,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741835_1011 (size=5153)
2024-12-06T01:52:52,062 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/.tmp/ns/002aab1280224de3979974891a190293
2024-12-06T01:52:52,071 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/.tmp/ns/002aab1280224de3979974891a190293 as hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/ns/002aab1280224de3979974891a190293
2024-12-06T01:52:52,078 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/ns/002aab1280224de3979974891a190293, entries=2, sequenceid=6, filesize=5.0 K
2024-12-06T01:52:52,079 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=6, compaction requested=false
2024-12-06T01:52:52,080 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T01:52:52,085 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-06T01:52:52,086 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:52:52,086 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:52:52,086 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733449972036Running coprocessor pre-close hooks at 1733449972036Disabling compacts and flushes for region at 1733449972036Disabling writes for close at 1733449972036Obtaining lock to block concurrent updates at 1733449972036Preparing flush snapshotting stores in 1588230740 at 1733449972036Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733449972037 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733449972038 (+1 ms)Flushing 1588230740/ns: creating writer at 1733449972038Flushing 1588230740/ns: appending metadata at 1733449972054 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733449972055 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20e50493: reopening flushed file at 1733449972070 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=6, compaction requested=false at 1733449972080 (+10 ms)Writing region close event to WAL at 1733449972081 (+1 ms)Running coprocessor post-close hooks at 1733449972086 (+5 ms)Closed at 1733449972086
2024-12-06T01:52:52,086 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:52:52,236 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,33527,1733449970971; all regions closed.
2024-12-06T01:52:52,237 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,237 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,237 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,237 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,238 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741834_1010 (size=1152)
2024-12-06T01:52:52,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741834_1010 (size=1152)
2024-12-06T01:52:52,244 DEBUG [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/oldWALs
2024-12-06T01:52:52,244 INFO  [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C33527%2C1733449970971.meta:.meta(num 1733449971879)
2024-12-06T01:52:52,244 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,245 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,245 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,245 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,245 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741833_1009 (size=93)
2024-12-06T01:52:52,248 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741833_1009 (size=93)
2024-12-06T01:52:52,251 DEBUG [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/oldWALs
2024-12-06T01:52:52,251 INFO  [RS:0;00c764f7c3aa:33527 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C33527%2C1733449970971:(num 1733449971524)
2024-12-06T01:52:52,251 DEBUG [RS:0;00c764f7c3aa:33527 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:52,251 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:52:52,251 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:52:52,251 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-06T01:52:52,251 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:52:52,251 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:52:52,251 INFO  [RS:0;00c764f7c3aa:33527 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33527
2024-12-06T01:52:52,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,33527,1733449970971
2024-12-06T01:52:52,254 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:52:52,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:52:52,257 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,33527,1733449970971]
2024-12-06T01:52:52,259 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,33527,1733449970971 already deleted, retry=false
2024-12-06T01:52:52,259 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,33527,1733449970971 expired; onlineServers=0
2024-12-06T01:52:52,259 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,42871,1733449970921' *****
2024-12-06T01:52:52,259 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:52:52,259 INFO  [M:0;00c764f7c3aa:42871 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:52:52,259 INFO  [M:0;00c764f7c3aa:42871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:52:52,259 DEBUG [M:0;00c764f7c3aa:42871 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:52:52,259 DEBUG [M:0;00c764f7c3aa:42871 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:52:52,259 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:52:52,259 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449971231 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449971231,5,FailOnTimeoutGroup]
2024-12-06T01:52:52,260 INFO  [M:0;00c764f7c3aa:42871 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:52:52,260 INFO  [M:0;00c764f7c3aa:42871 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:52:52,260 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449971232 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449971232,5,FailOnTimeoutGroup]
2024-12-06T01:52:52,260 DEBUG [M:0;00c764f7c3aa:42871 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:52:52,260 INFO  [M:0;00c764f7c3aa:42871 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:52:52,260 INFO  [M:0;00c764f7c3aa:42871 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:52:52,261 INFO  [M:0;00c764f7c3aa:42871 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:52:52,261 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:52:52,262 DEBUG [M:0;00c764f7c3aa:42871 {}] zookeeper.ZKUtil(347): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:52:52,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:52:52,262 WARN  [M:0;00c764f7c3aa:42871 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:52:52,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:52,263 INFO  [M:0;00c764f7c3aa:42871 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/.lastflushedseqids
2024-12-06T01:52:52,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741836_1012 (size=99)
2024-12-06T01:52:52,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741836_1012 (size=99)
2024-12-06T01:52:52,280 INFO  [M:0;00c764f7c3aa:42871 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:52:52,280 INFO  [M:0;00c764f7c3aa:42871 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:52:52,280 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:52:52,280 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:52,280 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:52,280 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:52:52,280 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:52,281 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-06T01:52:52,306 DEBUG [M:0;00c764f7c3aa:42871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16fc5e4189e0401e8581e7ab6e9afc11 is 82, key is hbase:meta,,1/info:regioninfo/1733449971913/Put/seqid=0
2024-12-06T01:52:52,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741837_1013 (size=5672)
2024-12-06T01:52:52,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741837_1013 (size=5672)
2024-12-06T01:52:52,324 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16fc5e4189e0401e8581e7ab6e9afc11
2024-12-06T01:52:52,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:52,356 INFO  [RS:0;00c764f7c3aa:33527 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:52:52,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33527-0x10065072db90001, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:52,356 INFO  [RS:0;00c764f7c3aa:33527 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,33527,1733449970971; zookeeper connection closed.
2024-12-06T01:52:52,357 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ca141 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ca141
2024-12-06T01:52:52,357 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T01:52:52,366 DEBUG [M:0;00c764f7c3aa:42871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1a625ea7d67d40b6afaffabf96cf0aeb is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733449971936/Put/seqid=0
2024-12-06T01:52:52,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741838_1014 (size=5275)
2024-12-06T01:52:52,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741838_1014 (size=5275)
2024-12-06T01:52:52,379 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1a625ea7d67d40b6afaffabf96cf0aeb
2024-12-06T01:52:52,404 DEBUG [M:0;00c764f7c3aa:42871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f2a0ebe523d44133ad38c8d7f0fd89da is 69, key is 00c764f7c3aa,33527,1733449970971/rs:state/1733449971341/Put/seqid=0
2024-12-06T01:52:52,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741839_1015 (size=5156)
2024-12-06T01:52:52,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741839_1015 (size=5156)
2024-12-06T01:52:52,418 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f2a0ebe523d44133ad38c8d7f0fd89da
2024-12-06T01:52:52,455 DEBUG [M:0;00c764f7c3aa:42871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e0cc3c195de54e41938a8c0401010494 is 52, key is load_balancer_on/state:d/1733449972028/Put/seqid=0
2024-12-06T01:52:52,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741840_1016 (size=5056)
2024-12-06T01:52:52,473 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e0cc3c195de54e41938a8c0401010494
2024-12-06T01:52:52,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741840_1016 (size=5056)
2024-12-06T01:52:52,486 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16fc5e4189e0401e8581e7ab6e9afc11 as hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16fc5e4189e0401e8581e7ab6e9afc11
2024-12-06T01:52:52,495 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16fc5e4189e0401e8581e7ab6e9afc11, entries=8, sequenceid=29, filesize=5.5 K
2024-12-06T01:52:52,496 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1a625ea7d67d40b6afaffabf96cf0aeb as hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1a625ea7d67d40b6afaffabf96cf0aeb
2024-12-06T01:52:52,505 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1a625ea7d67d40b6afaffabf96cf0aeb, entries=3, sequenceid=29, filesize=5.2 K
2024-12-06T01:52:52,507 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f2a0ebe523d44133ad38c8d7f0fd89da as hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f2a0ebe523d44133ad38c8d7f0fd89da
2024-12-06T01:52:52,521 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f2a0ebe523d44133ad38c8d7f0fd89da, entries=1, sequenceid=29, filesize=5.0 K
2024-12-06T01:52:52,523 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e0cc3c195de54e41938a8c0401010494 as hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e0cc3c195de54e41938a8c0401010494
2024-12-06T01:52:52,535 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39051/user/jenkins/test-data/699c66f6-d447-8059-d065-68aa98376f0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e0cc3c195de54e41938a8c0401010494, entries=1, sequenceid=29, filesize=4.9 K
2024-12-06T01:52:52,538 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 258ms, sequenceid=29, compaction requested=false
2024-12-06T01:52:52,547 INFO  [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:52,547 DEBUG [M:0;00c764f7c3aa:42871 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733449972280Disabling compacts and flushes for region at 1733449972280Disabling writes for close at 1733449972280Obtaining lock to block concurrent updates at 1733449972281 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733449972281Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733449972281Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733449972282 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733449972282Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733449972305 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733449972305Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733449972331 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733449972365 (+34 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733449972365Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733449972386 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733449972404 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733449972404Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733449972427 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733449972454 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733449972454Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@306cfd9e: reopening flushed file at 1733449972485 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79f3bb72: reopening flushed file at 1733449972495 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b80fd45: reopening flushed file at 1733449972506 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d26dd94: reopening flushed file at 1733449972521 (+15 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 258ms, sequenceid=29, compaction requested=false at 1733449972538 (+17 ms)Writing region close event to WAL at 1733449972547 (+9 ms)Closed at 1733449972547
2024-12-06T01:52:52,548 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,548 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,549 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,549 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,549 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:52:52,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741830_1006 (size=10311)
2024-12-06T01:52:52,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35601 is added to blk_1073741830_1006 (size=10311)
2024-12-06T01:52:52,554 INFO  [M:0;00c764f7c3aa:42871 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:52:52,554 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:52:52,554 INFO  [M:0;00c764f7c3aa:42871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42871
2024-12-06T01:52:52,555 INFO  [M:0;00c764f7c3aa:42871 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:52:52,657 INFO  [M:0;00c764f7c3aa:42871 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:52:52,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:52,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42871-0x10065072db90000, quorum=127.0.0.1:59806, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:52:52,660 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d2d0bcc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:52,660 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@57c7b86a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:52:52,660 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:52:52,660 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2502c8c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:52:52,661 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c9fe9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir/,STOPPED}
2024-12-06T01:52:52,662 WARN  [BP-1419783637-172.17.0.2-1733449970065 heartbeating to localhost/127.0.0.1:39051 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:52:52,662 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:52:52,662 WARN  [BP-1419783637-172.17.0.2-1733449970065 heartbeating to localhost/127.0.0.1:39051 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1419783637-172.17.0.2-1733449970065 (Datanode Uuid 29298d03-5e89-46c7-a60f-1eff67bd1793) service to localhost/127.0.0.1:39051
2024-12-06T01:52:52,662 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:52:52,663 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data3/current/BP-1419783637-172.17.0.2-1733449970065 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:52,663 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data4/current/BP-1419783637-172.17.0.2-1733449970065 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:52,664 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:52:52,666 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a787e38{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:52,666 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@144bd696{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:52:52,666 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:52:52,667 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22ba3420{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:52:52,667 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ca07ff3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir/,STOPPED}
2024-12-06T01:52:52,668 WARN  [BP-1419783637-172.17.0.2-1733449970065 heartbeating to localhost/127.0.0.1:39051 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:52:52,668 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:52:52,668 WARN  [BP-1419783637-172.17.0.2-1733449970065 heartbeating to localhost/127.0.0.1:39051 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1419783637-172.17.0.2-1733449970065 (Datanode Uuid d99a1816-54c3-4d29-b749-75da1adfc1b8) service to localhost/127.0.0.1:39051
2024-12-06T01:52:52,668 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:52:52,669 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data1/current/BP-1419783637-172.17.0.2-1733449970065 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:52,669 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/cluster_1242ec79-76a8-1c1c-d0e6-cb1ca7cee545/data/data2/current/BP-1419783637-172.17.0.2-1733449970065 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:52:52,670 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:52:52,676 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5316f6d0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:52:52,676 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ced705f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:52:52,676 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:52:52,677 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73737af2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:52:52,677 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@246cb388{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir/,STOPPED}
2024-12-06T01:52:52,687 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:52:52,705 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:52:52,705 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.log.dir so I do NOT create it in target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a07fe9c-d5ea-0f8f-f280-7716bbe93ffe/hadoop.tmp.dir so I do NOT create it in target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294, deleteOnExit=true
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/test.cache.data in system properties and HBase conf
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:52:52,706 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:52:52,707 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:52:52,707 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:52:52,708 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:52:52,722 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:52:52,796 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:52,802 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:52:52,804 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:52:52,804 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:52:52,804 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:52:52,805 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:52,806 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9fcb45e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:52:52,807 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e8e5e50{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:52:52,930 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70e75057{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-40885-hadoop-hdfs-3_4_1-tests_jar-_-any-14711876440083661747/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:52:52,931 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a936a30{HTTP/1.1, (http/1.1)}{localhost:40885}
2024-12-06T01:52:52,931 INFO  [Time-limited test {}] server.Server(415): Started @106199ms
2024-12-06T01:52:52,946 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:52:53,021 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:53,026 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:52:53,026 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:52:53,026 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:52:53,027 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:52:53,027 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@284f089c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:52:53,028 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@517471d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:52:53,153 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7049c77d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-41203-hadoop-hdfs-3_4_1-tests_jar-_-any-9670759478449080503/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:53,153 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@18e85f30{HTTP/1.1, (http/1.1)}{localhost:41203}
2024-12-06T01:52:53,153 INFO  [Time-limited test {}] server.Server(415): Started @106421ms
2024-12-06T01:52:53,155 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:52:53,201 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:52:53,210 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:52:53,211 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:52:53,211 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:52:53,211 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:52:53,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e8d53e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:52:53,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5057e60f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:52:53,270 WARN  [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data1/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:53,270 WARN  [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data2/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:53,294 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:52:53,297 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x90b12f9506a4820e with lease ID 0x951377814a1ffe6d: Processing first storage report for DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e from datanode DatanodeRegistration(127.0.0.1:37205, datanodeUuid=fe13279f-a1fe-4591-82ab-80a744b161f0, infoPort=46153, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:52:53,297 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90b12f9506a4820e with lease ID 0x951377814a1ffe6d: from storage DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e node DatanodeRegistration(127.0.0.1:37205, datanodeUuid=fe13279f-a1fe-4591-82ab-80a744b161f0, infoPort=46153, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:52:53,297 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x90b12f9506a4820e with lease ID 0x951377814a1ffe6d: Processing first storage report for DS-163e30b5-d490-40c9-a473-bf008c201e28 from datanode DatanodeRegistration(127.0.0.1:37205, datanodeUuid=fe13279f-a1fe-4591-82ab-80a744b161f0, infoPort=46153, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:52:53,297 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90b12f9506a4820e with lease ID 0x951377814a1ffe6d: from storage DS-163e30b5-d490-40c9-a473-bf008c201e28 node DatanodeRegistration(127.0.0.1:37205, datanodeUuid=fe13279f-a1fe-4591-82ab-80a744b161f0, infoPort=46153, infoSecurePort=0, ipcPort=37741, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:52:53,365 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:52:53,379 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6710cf51{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-42875-hadoop-hdfs-3_4_1-tests_jar-_-any-13451252931702438098/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:52:53,380 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6954ffea{HTTP/1.1, (http/1.1)}{localhost:42875}
2024-12-06T01:52:53,380 INFO  [Time-limited test {}] server.Server(415): Started @106648ms
2024-12-06T01:52:53,382 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:52:53,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:52:53,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:52:53,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T01:52:53,407 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-06T01:52:53,472 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data3/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:53,472 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data4/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:52:53,496 WARN  [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:52:53,499 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x722b70ea2c2edbeb with lease ID 0x951377814a1ffe6e: Processing first storage report for DS-9b14a558-ddd4-4393-8642-eb18a93bb91e from datanode DatanodeRegistration(127.0.0.1:45875, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=42275, infoSecurePort=0, ipcPort=34239, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:52:53,499 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x722b70ea2c2edbeb with lease ID 0x951377814a1ffe6e: from storage DS-9b14a558-ddd4-4393-8642-eb18a93bb91e node DatanodeRegistration(127.0.0.1:45875, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=42275, infoSecurePort=0, ipcPort=34239, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:52:53,499 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x722b70ea2c2edbeb with lease ID 0x951377814a1ffe6e: Processing first storage report for DS-b4f93e80-73e0-43f3-9b97-a2100a700915 from datanode DatanodeRegistration(127.0.0.1:45875, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=42275, infoSecurePort=0, ipcPort=34239, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:52:53,499 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x722b70ea2c2edbeb with lease ID 0x951377814a1ffe6e: from storage DS-b4f93e80-73e0-43f3-9b97-a2100a700915 node DatanodeRegistration(127.0.0.1:45875, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=42275, infoSecurePort=0, ipcPort=34239, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:52:53,510 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84
2024-12-06T01:52:53,512 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/zookeeper_0, clientPort=53502, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:52:53,513 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53502
2024-12-06T01:52:53,514 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,516 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:52:53,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:52:53,527 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c with version=8
2024-12-06T01:52:53,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:52:53,529 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:52:53,529 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:53,529 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:53,530 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:52:53,530 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:53,530 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:52:53,530 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:52:53,530 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:52:53,530 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36647
2024-12-06T01:52:53,532 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36647 connecting to ZooKeeper ensemble=127.0.0.1:53502
2024-12-06T01:52:53,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:366470x0, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:52:53,538 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36647-0x100650737ee0000 connected
2024-12-06T01:52:53,552 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,555 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:53,558 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c, hbase.cluster.distributed=false
2024-12-06T01:52:53,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:52:53,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36647
2024-12-06T01:52:53,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36647
2024-12-06T01:52:53,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36647
2024-12-06T01:52:53,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36647
2024-12-06T01:52:53,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36647
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:52:53,579 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:52:53,580 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45229
2024-12-06T01:52:53,581 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45229 connecting to ZooKeeper ensemble=127.0.0.1:53502
2024-12-06T01:52:53,582 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,584 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452290x0, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:52:53,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:52:53,590 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45229-0x100650737ee0001 connected
2024-12-06T01:52:53,590 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:52:53,591 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:52:53,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:52:53,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:52:53,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45229
2024-12-06T01:52:53,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45229
2024-12-06T01:52:53,598 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45229
2024-12-06T01:52:53,598 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45229
2024-12-06T01:52:53,598 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45229
2024-12-06T01:52:53,618 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:36647
2024-12-06T01:52:53,619 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:53,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:53,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:53,621 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:53,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:52:53,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:53,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:53,623 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:52:53,624 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,36647,1733449973529 from backup master directory
2024-12-06T01:52:53,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:53,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:53,626 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:52:53,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:52:53,626 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:53,631 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/hbase.id] with ID: 1a9f906a-3039-47f6-9c9a-2ff70f072de0
2024-12-06T01:52:53,631 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/.tmp/hbase.id
2024-12-06T01:52:53,640 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:52:53,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:52:53,641 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/.tmp/hbase.id]:[hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/hbase.id]
2024-12-06T01:52:53,655 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:53,655 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:52:53,657 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-06T01:52:53,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:53,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:53,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:52:53,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:52:53,672 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:52:53,673 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:52:53,673 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:53,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:52:53,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:52:53,686 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store
2024-12-06T01:52:53,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:52:53,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:52:54,101 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:54,101 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:52:54,101 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:54,101 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:54,101 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:52:54,101 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:54,101 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:52:54,101 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733449974101Disabling compacts and flushes for region at 1733449974101Disabling writes for close at 1733449974101Writing region close event to WAL at 1733449974101Closed at 1733449974101
2024-12-06T01:52:54,103 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/.initializing
2024-12-06T01:52:54,103 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:54,106 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C36647%2C1733449973529, suffix=, logDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529, archiveDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/oldWALs, maxLogs=10
2024-12-06T01:52:54,107 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C36647%2C1733449973529.1733449974106
2024-12-06T01:52:54,116 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106
2024-12-06T01:52:54,118 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46153:46153),(127.0.0.1/127.0.0.1:42275:42275)]
2024-12-06T01:52:54,118 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:52:54,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:54,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,120 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:54,122 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:52:54,122 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,123 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,123 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:54,124 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:52:54,124 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,125 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:54,125 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:54,127 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:52:54,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,127 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:54,127 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:52:54,128 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:52:54,129 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,129 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:54,129 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,130 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,130 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,132 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,132 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,133 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:52:54,134 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:52:54,137 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:52:54,137 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791387, jitterRate=0.006301671266555786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:52:54,139 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733449974119Initializing all the Stores at 1733449974120 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974120Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449974120Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449974120Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449974120Cleaning up temporary data from old regions at 1733449974132 (+12 ms)Region opened successfully at 1733449974138 (+6 ms)
2024-12-06T01:52:54,139 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:52:54,142 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7204a13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:52:54,144 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:52:54,144 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:52:54,144 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:52:54,144 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:52:54,145 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T01:52:54,145 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T01:52:54,145 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:52:54,148 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:52:54,148 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:52:54,150 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:52:54,150 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:52:54,151 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:52:54,152 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:52:54,152 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:52:54,153 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:52:54,155 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:52:54,156 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:52:54,157 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:52:54,159 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:52:54,160 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:52:54,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:54,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:52:54,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,162 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,36647,1733449973529, sessionid=0x100650737ee0000, setting cluster-up flag (Was=false)
2024-12-06T01:52:54,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,169 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:52:54,170 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:54,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,178 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:52:54,179 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:54,180 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:52:54,182 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:54,182 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:52:54,183 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:52:54,183 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,36647,1733449973529
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:52:54,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733450004186
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:52:54,186 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:54,186 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:52:54,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,187 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:52:54,187 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:52:54,187 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:52:54,187 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:52:54,187 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:52:54,188 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,188 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449974187,5,FailOnTimeoutGroup]
2024-12-06T01:52:54,188 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449974188,5,FailOnTimeoutGroup]
2024-12-06T01:52:54,188 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,188 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:52:54,188 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:52:54,188 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,188 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:52:54,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:52:54,196 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:52:54,196 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c
2024-12-06T01:52:54,201 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(746): ClusterId : 1a9f906a-3039-47f6-9c9a-2ff70f072de0
2024-12-06T01:52:54,201 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:52:54,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:52:54,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:52:54,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:54,204 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:52:54,204 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:52:54,205 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:52:54,206 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:52:54,207 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:52:54,207 DEBUG [RS:0;00c764f7c3aa:45229 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf1fb71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:52:54,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,207 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,208 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:52:54,209 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:52:54,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,210 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,210 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:52:54,211 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:52:54,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,212 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,212 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:52:54,213 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:52:54,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,214 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:52:54,214 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740
2024-12-06T01:52:54,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740
2024-12-06T01:52:54,216 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:52:54,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:52:54,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:52:54,217 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:52:54,218 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:52:54,221 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:52:54,222 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850920, jitterRate=0.08200138807296753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:52:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733449974204Initializing all the Stores at 1733449974205 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974205Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974205Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449974205Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974205Cleaning up temporary data from old regions at 1733449974217 (+12 ms)Region opened successfully at 1733449974223 (+6 ms)
2024-12-06T01:52:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:52:54,223 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:52:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:52:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:52:54,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:52:54,224 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:52:54,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733449974223Disabling compacts and flushes for region at 1733449974223Disabling writes for close at 1733449974223Writing region close event to WAL at 1733449974224 (+1 ms)Closed at 1733449974224
2024-12-06T01:52:54,224 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:45229
2024-12-06T01:52:54,225 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:52:54,225 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:52:54,225 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:52:54,226 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,36647,1733449973529 with port=45229, startcode=1733449973578
2024-12-06T01:52:54,226 DEBUG [RS:0;00c764f7c3aa:45229 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:52:54,226 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:54,226 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:52:54,226 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:52:54,226 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:52:54,229 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57243, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:52:54,229 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:52:54,229 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,230 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36647 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,231 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:52:54,232 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c
2024-12-06T01:52:54,232 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41913
2024-12-06T01:52:54,232 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:52:54,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:52:54,235 DEBUG [RS:0;00c764f7c3aa:45229 {}] zookeeper.ZKUtil(111): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,235 WARN  [RS:0;00c764f7c3aa:45229 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:52:54,235 INFO  [RS:0;00c764f7c3aa:45229 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:54,235 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,235 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,45229,1733449973578]
2024-12-06T01:52:54,241 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:52:54,244 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:52:54,244 INFO  [RS:0;00c764f7c3aa:45229 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:52:54,244 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,244 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:52:54,245 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:52:54,246 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,246 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,247 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:54,247 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:52:54,247 DEBUG [RS:0;00c764f7c3aa:45229 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:52:54,248 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,248 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,248 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,248 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,248 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,248 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,45229,1733449973578-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:52:54,272 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:52:54,272 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,45229,1733449973578-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,272 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,272 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.Replication(171): 00c764f7c3aa,45229,1733449973578 started
2024-12-06T01:52:54,287 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,288 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,45229,1733449973578, RpcServer on 00c764f7c3aa/172.17.0.2:45229, sessionid=0x100650737ee0001
2024-12-06T01:52:54,288 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:52:54,288 DEBUG [RS:0;00c764f7c3aa:45229 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,288 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,45229,1733449973578'
2024-12-06T01:52:54,288 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:52:54,289 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:52:54,289 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:52:54,289 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:52:54,289 DEBUG [RS:0;00c764f7c3aa:45229 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,289 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,45229,1733449973578'
2024-12-06T01:52:54,289 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:52:54,290 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:52:54,290 DEBUG [RS:0;00c764f7c3aa:45229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:52:54,291 INFO  [RS:0;00c764f7c3aa:45229 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:52:54,291 INFO  [RS:0;00c764f7c3aa:45229 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:52:54,382 WARN  [00c764f7c3aa:36647 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:52:54,393 INFO  [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C45229%2C1733449973578, suffix=, logDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578, archiveDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs, maxLogs=32
2024-12-06T01:52:54,422 INFO  [RS:0;00c764f7c3aa:45229 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.1733449974421
2024-12-06T01:52:54,431 INFO  [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421
2024-12-06T01:52:54,435 DEBUG [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46153:46153),(127.0.0.1/127.0.0.1:42275:42275)]
2024-12-06T01:52:54,632 DEBUG [00c764f7c3aa:36647 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:52:54,633 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,634 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,45229,1733449973578, state=OPENING
2024-12-06T01:52:54,636 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:52:54,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:52:54,638 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:52:54,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:54,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:54,639 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45229,1733449973578}]
2024-12-06T01:52:54,739 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:52:54,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:52:54,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:52:54,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:52:54,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:52:54,792 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:52:54,795 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:52:54,799 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:52:54,799 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:54,801 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C45229%2C1733449973578.meta, suffix=.meta, logDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578, archiveDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs, maxLogs=32
2024-12-06T01:52:54,801 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
2024-12-06T01:52:54,807 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
2024-12-06T01:52:54,807 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46153:46153),(127.0.0.1/127.0.0.1:42275:42275)]
2024-12-06T01:52:54,808 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:52:54,809 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:52:54,809 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:52:54,809 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:52:54,809 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:52:54,809 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:54,810 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:52:54,810 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:52:54,811 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:52:54,812 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:52:54,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,813 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,813 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:52:54,814 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:52:54,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,814 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,814 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:52:54,815 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:52:54,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,815 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,815 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:52:54,816 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:52:54,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:54,817 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:52:54,817 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:52:54,818 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740
2024-12-06T01:52:54,819 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740
2024-12-06T01:52:54,821 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:52:54,821 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:52:54,821 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:52:54,823 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:52:54,824 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842280, jitterRate=0.07101503014564514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:52:54,824 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:52:54,826 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733449974810Writing region info on filesystem at 1733449974810Initializing all the Stores at 1733449974811 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974811Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974811Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449974811Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449974811Cleaning up temporary data from old regions at 1733449974821 (+10 ms)Running coprocessor post-open hooks at 1733449974824 (+3 ms)Region opened successfully at 1733449974826 (+2 ms)
2024-12-06T01:52:54,827 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733449974792
2024-12-06T01:52:54,830 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:52:54,830 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:52:54,831 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,832 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,45229,1733449973578, state=OPEN
2024-12-06T01:52:54,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:52:54,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:52:54,836 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:54,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:54,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:52:54,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:52:54,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45229,1733449973578 in 198 msec
2024-12-06T01:52:54,843 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:52:54,843 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 614 msec
2024-12-06T01:52:54,844 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:52:54,845 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:52:54,846 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:52:54,846 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45229,1733449973578, seqNum=-1]
2024-12-06T01:52:54,846 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:52:54,848 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37549, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:52:54,854 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 671 msec
2024-12-06T01:52:54,854 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733449974854, completionTime=-1
2024-12-06T01:52:54,854 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:52:54,854 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:52:54,856 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:52:54,856 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733450034856
2024-12-06T01:52:54,856 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733450094856
2024-12-06T01:52:54,856 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T01:52:54,857 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36647,1733449973529-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,857 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36647,1733449973529-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,857 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36647,1733449973529-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,857 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:36647, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,857 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,858 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,859 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:52:54,861 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.235sec
2024-12-06T01:52:54,861 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:52:54,861 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:52:54,862 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:52:54,862 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:52:54,862 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:52:54,862 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36647,1733449973529-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:52:54,862 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36647,1733449973529-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:52:54,865 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:52:54,865 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:52:54,865 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36647,1733449973529-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:54,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76394935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:52:54,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,36647,-1 for getting cluster id
2024-12-06T01:52:54,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:52:54,925 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1a9f906a-3039-47f6-9c9a-2ff70f072de0'
2024-12-06T01:52:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:52:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1a9f906a-3039-47f6-9c9a-2ff70f072de0"

2024-12-06T01:52:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22464501, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:52:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,36647,-1]
2024-12-06T01:52:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:52:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:52:54,928 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:52:54,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17308bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:52:54,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:52:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45229,1733449973578, seqNum=-1]
2024-12-06T01:52:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:52:54,932 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:52:54,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:54,934 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:54,937 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:52:54,954 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:52:54,955 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37779
2024-12-06T01:52:54,956 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37779 connecting to ZooKeeper ensemble=127.0.0.1:53502
2024-12-06T01:52:54,957 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:54,958 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:52:54,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377790x0, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:52:54,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:377790x0, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-06T01:52:54,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37779-0x100650737ee0002 connected
2024-12-06T01:52:54,963 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-06T01:52:54,963 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:52:54,964 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:52:54,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:52:54,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:52:54,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37779
2024-12-06T01:52:54,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37779
2024-12-06T01:52:54,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37779
2024-12-06T01:52:54,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37779
2024-12-06T01:52:54,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37779
2024-12-06T01:52:54,969 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(746): ClusterId : 1a9f906a-3039-47f6-9c9a-2ff70f072de0
2024-12-06T01:52:54,969 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:52:54,971 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:52:54,972 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:52:54,974 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:52:54,974 DEBUG [RS:1;00c764f7c3aa:37779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d7d204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:52:54,987 DEBUG [RS:1;00c764f7c3aa:37779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;00c764f7c3aa:37779
2024-12-06T01:52:54,987 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:52:54,987 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:52:54,987 DEBUG [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:52:54,988 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,36647,1733449973529 with port=37779, startcode=1733449974953
2024-12-06T01:52:54,988 DEBUG [RS:1;00c764f7c3aa:37779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:52:54,990 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33195, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:52:54,990 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,37779,1733449974953
2024-12-06T01:52:54,991 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36647 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,37779,1733449974953
2024-12-06T01:52:54,992 DEBUG [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c
2024-12-06T01:52:54,993 DEBUG [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41913
2024-12-06T01:52:54,993 DEBUG [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:52:54,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:52:54,995 DEBUG [RS:1;00c764f7c3aa:37779 {}] zookeeper.ZKUtil(111): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,37779,1733449974953
2024-12-06T01:52:54,995 WARN  [RS:1;00c764f7c3aa:37779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:52:54,995 INFO  [RS:1;00c764f7c3aa:37779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:52:54,995 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,37779,1733449974953]
2024-12-06T01:52:54,995 DEBUG [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953
2024-12-06T01:52:54,999 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:52:55,001 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:52:55,001 INFO  [RS:1;00c764f7c3aa:37779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:52:55,002 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,002 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:52:55,003 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:52:55,003 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,003 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,004 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:52:55,004 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:52:55,004 DEBUG [RS:1;00c764f7c3aa:37779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:52:55,004 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,004 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,004 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,004 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,004 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,004 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37779,1733449974953-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:52:55,020 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:52:55,021 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37779,1733449974953-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,021 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,021 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.Replication(171): 00c764f7c3aa,37779,1733449974953 started
2024-12-06T01:52:55,035 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:52:55,035 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,37779,1733449974953, RpcServer on 00c764f7c3aa/172.17.0.2:37779, sessionid=0x100650737ee0002
2024-12-06T01:52:55,035 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:52:55,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;00c764f7c3aa:37779,5,FailOnTimeoutGroup]
2024-12-06T01:52:55,035 DEBUG [RS:1;00c764f7c3aa:37779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,37779,1733449974953
2024-12-06T01:52:55,035 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,37779,1733449974953'
2024-12-06T01:52:55,035 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:52:55,035 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-06T01:52:55,036 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T01:52:55,036 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:52:55,036 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:52:55,036 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:52:55,036 DEBUG [RS:1;00c764f7c3aa:37779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,37779,1733449974953
2024-12-06T01:52:55,036 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,37779,1733449974953'
2024-12-06T01:52:55,036 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:52:55,037 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:52:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,36647,1733449973529
2024-12-06T01:52:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@71fb2c26
2024-12-06T01:52:55,037 DEBUG [RS:1;00c764f7c3aa:37779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:52:55,037 INFO  [RS:1;00c764f7c3aa:37779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:52:55,037 INFO  [RS:1;00c764f7c3aa:37779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:52:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T01:52:55,039 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T01:52:55,040 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T01:52:55,040 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T01:52:55,040 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:52:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T01:52:55,043 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T01:52:55,043 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:55,043 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-06T01:52:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:52:55,045 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T01:52:55,054 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741835_1011 (size=393)
2024-12-06T01:52:55,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741835_1011 (size=393)
2024-12-06T01:52:55,057 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ce716e29eafea16423da07d2956326df, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c
2024-12-06T01:52:55,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45875 is added to blk_1073741836_1012 (size=76)
2024-12-06T01:52:55,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741836_1012 (size=76)
2024-12-06T01:52:55,065 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:55,065 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing ce716e29eafea16423da07d2956326df, disabling compactions & flushes
2024-12-06T01:52:55,065 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,065 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,065 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df. after waiting 0 ms
2024-12-06T01:52:55,065 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,065 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,065 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for ce716e29eafea16423da07d2956326df:
Waiting for close lock at 1733449975065Disabling compacts and flushes for region at 1733449975065Disabling writes for close at 1733449975065Writing region close event to WAL at 1733449975065Closed at 1733449975065
2024-12-06T01:52:55,067 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T01:52:55,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733449975067"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733449975067"}]},"ts":"1733449975067"}
2024-12-06T01:52:55,071 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T01:52:55,073 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T01:52:55,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449975073"}]},"ts":"1733449975073"}
2024-12-06T01:52:55,076 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-06T01:52:55,076 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ce716e29eafea16423da07d2956326df, ASSIGN}]
2024-12-06T01:52:55,078 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ce716e29eafea16423da07d2956326df, ASSIGN
2024-12-06T01:52:55,079 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ce716e29eafea16423da07d2956326df, ASSIGN; state=OFFLINE, location=00c764f7c3aa,45229,1733449973578; forceNewPlan=false, retain=false
2024-12-06T01:52:55,140 INFO  [RS:1;00c764f7c3aa:37779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C37779%2C1733449974953, suffix=, logDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953, archiveDir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs, maxLogs=32
2024-12-06T01:52:55,141 INFO  [RS:1;00c764f7c3aa:37779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C37779%2C1733449974953.1733449975140
2024-12-06T01:52:55,149 INFO  [RS:1;00c764f7c3aa:37779 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
2024-12-06T01:52:55,154 DEBUG [RS:1;00c764f7c3aa:37779 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42275:42275),(127.0.0.1/127.0.0.1:46153:46153)]
2024-12-06T01:52:55,230 INFO  [00c764f7c3aa:36647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-06T01:52:55,231 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ce716e29eafea16423da07d2956326df, regionState=OPENING, regionLocation=00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:55,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ce716e29eafea16423da07d2956326df, ASSIGN because future has completed
2024-12-06T01:52:55,234 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce716e29eafea16423da07d2956326df, server=00c764f7c3aa,45229,1733449973578}]
2024-12-06T01:52:55,392 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,393 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ce716e29eafea16423da07d2956326df, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:52:55,394 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,394 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:52:55,394 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,394 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,396 INFO  [StoreOpener-ce716e29eafea16423da07d2956326df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ce716e29eafea16423da07d2956326df 
2024-12-06T01:52:55,398 INFO  [StoreOpener-ce716e29eafea16423da07d2956326df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce716e29eafea16423da07d2956326df columnFamilyName info
2024-12-06T01:52:55,398 DEBUG [StoreOpener-ce716e29eafea16423da07d2956326df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:52:55,398 INFO  [StoreOpener-ce716e29eafea16423da07d2956326df-1 {}] regionserver.HStore(327): Store=ce716e29eafea16423da07d2956326df/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:52:55,398 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,399 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,400 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,400 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,400 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,402 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,405 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:52:55,405 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ce716e29eafea16423da07d2956326df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809271, jitterRate=0.02904212474822998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:52:55,405 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce716e29eafea16423da07d2956326df
2024-12-06T01:52:55,406 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ce716e29eafea16423da07d2956326df:
Running coprocessor pre-open hook at 1733449975394Writing region info on filesystem at 1733449975394Initializing all the Stores at 1733449975395 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449975395Cleaning up temporary data from old regions at 1733449975400 (+5 ms)Running coprocessor post-open hooks at 1733449975405 (+5 ms)Region opened successfully at 1733449975406 (+1 ms)
2024-12-06T01:52:55,407 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df., pid=6, masterSystemTime=1733449975387
2024-12-06T01:52:55,410 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,410 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:52:55,411 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ce716e29eafea16423da07d2956326df, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,45229,1733449973578
2024-12-06T01:52:55,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce716e29eafea16423da07d2956326df, server=00c764f7c3aa,45229,1733449973578 because future has completed
2024-12-06T01:52:55,418 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T01:52:55,418 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ce716e29eafea16423da07d2956326df, server=00c764f7c3aa,45229,1733449973578 in 181 msec
2024-12-06T01:52:55,421 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T01:52:55,422 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ce716e29eafea16423da07d2956326df, ASSIGN in 342 msec
2024-12-06T01:52:55,423 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T01:52:55,423 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449975423"}]},"ts":"1733449975423"}
2024-12-06T01:52:55,425 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-06T01:52:55,426 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T01:52:55,429 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 386 msec
2024-12-06T01:53:00,311 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:53:00,313 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:00,328 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:00,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:00,332 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:00,345 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-06T01:53:03,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:53:03,405 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T01:53:03,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T01:53:03,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-06T01:53:03,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:53:03,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T01:53:03,407 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T01:53:03,407 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-06T01:53:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:53:05,149 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-06T01:53:05,149 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-06T01:53:05,153 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T01:53:05,153 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:05,166 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:05,169 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:05,169 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:05,169 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:05,169 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:53:05,170 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b3a85a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:05,170 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a3456d5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:05,284 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@621d6abe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-43761-hadoop-hdfs-3_4_1-tests_jar-_-any-1781842396558467797/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:05,285 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@511ebf5d{HTTP/1.1, (http/1.1)}{localhost:43761}
2024-12-06T01:53:05,285 INFO  [Time-limited test {}] server.Server(415): Started @118553ms
2024-12-06T01:53:05,286 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:05,321 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:05,326 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:05,326 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:05,326 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:05,327 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:53:05,327 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6413b81{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:05,328 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@782c3e10{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:05,386 WARN  [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data5/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:05,387 WARN  [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data6/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:05,410 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:05,413 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x77448f54b1296336 with lease ID 0x951377814a1ffe6f: Processing first storage report for DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2 from datanode DatanodeRegistration(127.0.0.1:35333, datanodeUuid=8d884b35-0de4-4814-a84f-1b0422766da0, infoPort=39777, infoSecurePort=0, ipcPort=34871, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:53:05,413 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77448f54b1296336 with lease ID 0x951377814a1ffe6f: from storage DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2 node DatanodeRegistration(127.0.0.1:35333, datanodeUuid=8d884b35-0de4-4814-a84f-1b0422766da0, infoPort=39777, infoSecurePort=0, ipcPort=34871, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:53:05,413 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x77448f54b1296336 with lease ID 0x951377814a1ffe6f: Processing first storage report for DS-b0ea36fd-999d-4e80-a24a-88f84e067209 from datanode DatanodeRegistration(127.0.0.1:35333, datanodeUuid=8d884b35-0de4-4814-a84f-1b0422766da0, infoPort=39777, infoSecurePort=0, ipcPort=34871, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:53:05,413 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77448f54b1296336 with lease ID 0x951377814a1ffe6f: from storage DS-b0ea36fd-999d-4e80-a24a-88f84e067209 node DatanodeRegistration(127.0.0.1:35333, datanodeUuid=8d884b35-0de4-4814-a84f-1b0422766da0, infoPort=39777, infoSecurePort=0, ipcPort=34871, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:05,461 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@63e7f9e9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-37997-hadoop-hdfs-3_4_1-tests_jar-_-any-4304778444084115749/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:05,462 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3cbdbe64{HTTP/1.1, (http/1.1)}{localhost:37997}
2024-12-06T01:53:05,462 INFO  [Time-limited test {}] server.Server(415): Started @118730ms
2024-12-06T01:53:05,463 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:05,515 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:05,519 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:05,525 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:05,525 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:05,525 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:53:05,526 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@90666a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:05,527 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47917ab3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:05,578 WARN  [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data7/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:05,578 WARN  [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data8/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:05,605 WARN  [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:05,608 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9ac36577e159d48c with lease ID 0x951377814a1ffe70: Processing first storage report for DS-2af26174-4be5-43c1-a63a-7d2eb15097cf from datanode DatanodeRegistration(127.0.0.1:32769, datanodeUuid=e5478376-a8e8-4192-a861-8710614e826e, infoPort=38097, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:53:05,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9ac36577e159d48c with lease ID 0x951377814a1ffe70: from storage DS-2af26174-4be5-43c1-a63a-7d2eb15097cf node DatanodeRegistration(127.0.0.1:32769, datanodeUuid=e5478376-a8e8-4192-a861-8710614e826e, infoPort=38097, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:05,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9ac36577e159d48c with lease ID 0x951377814a1ffe70: Processing first storage report for DS-01d4e133-867c-4790-a526-a9fb4bca77b2 from datanode DatanodeRegistration(127.0.0.1:32769, datanodeUuid=e5478376-a8e8-4192-a861-8710614e826e, infoPort=38097, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:53:05,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9ac36577e159d48c with lease ID 0x951377814a1ffe70: from storage DS-01d4e133-867c-4790-a526-a9fb4bca77b2 node DatanodeRegistration(127.0.0.1:32769, datanodeUuid=e5478376-a8e8-4192-a861-8710614e826e, infoPort=38097, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:05,651 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d93430d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-34347-hadoop-hdfs-3_4_1-tests_jar-_-any-11314982804009076174/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:05,652 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@241fb63b{HTTP/1.1, (http/1.1)}{localhost:34347}
2024-12-06T01:53:05,652 INFO  [Time-limited test {}] server.Server(415): Started @118919ms
2024-12-06T01:53:05,653 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:05,757 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:05,758 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10/current/BP-811587157-172.17.0.2-1733449972741/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:05,774 WARN  [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:05,777 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x92768c2224c7584 with lease ID 0x951377814a1ffe71: Processing first storage report for DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1 from datanode DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:53:05,777 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x92768c2224c7584 with lease ID 0x951377814a1ffe71: from storage DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1 node DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:05,778 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x92768c2224c7584 with lease ID 0x951377814a1ffe71: Processing first storage report for DS-723ce39d-ccf6-4611-9483-c10a4b80781b from datanode DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741)
2024-12-06T01:53:05,778 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x92768c2224c7584 with lease ID 0x951377814a1ffe71: from storage DS-723ce39d-ccf6-4611-9483-c10a4b80781b node DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:05,877 WARN  [ResponseProcessor for block BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,877 WARN  [ResponseProcessor for block BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,877 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140 block BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:05,878 WARN  [ResponseProcessor for block BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,878 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 block BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:05,878 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta block BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:05,878 WARN  [PacketResponder: BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45875] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,879 WARN  [ResponseProcessor for block BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,878 WARN  [PacketResponder: BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45875] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,879 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:39152 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39152 dst: /127.0.0.1:37205
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,879 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:39146 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39146 dst: /127.0.0.1:37205
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,879 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1116501813_22 at /127.0.0.1:39666 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:45875:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39666 dst: /127.0.0.1:45875
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,880 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 block BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:05,880 WARN  [PacketResponder: BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45875] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,880 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:39634 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45875:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39634 dst: /127.0.0.1:45875
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,881 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1116501813_22 at /127.0.0.1:39176 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39176 dst: /127.0.0.1:37205
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,881 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:39632 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45875:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39632 dst: /127.0.0.1:45875
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,881 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-819121948_22 at /127.0.0.1:39124 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39124 dst: /127.0.0.1:37205
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,881 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-819121948_22 at /127.0.0.1:39608 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45875:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39608 dst: /127.0.0.1:45875
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,882 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6710cf51{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:05,883 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6954ffea{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:05,883 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:05,883 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5057e60f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:05,883 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e8d53e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:05,885 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:05,885 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-811587157-172.17.0.2-1733449972741 (Datanode Uuid feb48222-10f2-4cb1-95f9-4dc13a60c6e4) service to localhost/127.0.0.1:41913
2024-12-06T01:53:05,886 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data3/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:05,886 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data4/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:05,886 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:05,886 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:05,887 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:05,887 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 block BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,887 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 block BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,887 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140 block BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,888 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@7c71764f {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing unknown operation  src: /127.0.0.1:45670 dst: /127.0.0.1:37205
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:05,888 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta block BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7049c77d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:05,892 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@18e85f30{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:05,892 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:05,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@517471d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:05,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@284f089c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:05,894 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:05,894 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:05,894 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-811587157-172.17.0.2-1733449972741 (Datanode Uuid fe13279f-a1fe-4591-82ab-80a744b161f0) service to localhost/127.0.0.1:41913
2024-12-06T01:53:05,894 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:05,895 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data1/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:05,895 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data2/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:05,895 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df., hostname=00c764f7c3aa,45229,1733449973578, seqNum=2]
2024-12-06T01:53:05,903 ERROR [FSHLog-0-hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c-prefix:00c764f7c3aa,45229,1733449973578 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,904 WARN  [FSHLog-0-hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c-prefix:00c764f7c3aa,45229,1733449973578 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,904 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,904 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C45229%2C1733449973578:(num 1733449974421) roll requested
2024-12-06T01:53:05,904 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.1733449985904
2024-12-06T01:53:05,907 WARN  [Thread-902 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,908 WARN  [Thread-902 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:05,908 WARN  [Thread-902 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741838_1018
2024-12-06T01:53:05,910 WARN  [Thread-902 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:05,917 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:05,917 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:05,917 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:05,917 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:05,917 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:05,917 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449985904
2024-12-06T01:53:05,918 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,918 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:05,919 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38097:38097),(127.0.0.1/127.0.0.1:39777:39777)]
2024-12-06T01:53:05,919 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:05,919 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-06T01:53:05,919 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-06T01:53:05,919 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421
2024-12-06T01:53:05,922 WARN  [IPC Server handler 1 on default port 41913 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009
2024-12-06T01:53:05,926 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 after 5ms
2024-12-06T01:53:07,005 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:07,674 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:07,919 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:07,921 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449985904
2024-12-06T01:53:07,921 WARN  [ResponseProcessor for block BP-811587157-172.17.0.2-1733449972741:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-811587157-172.17.0.2-1733449972741:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:07,922 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449985904 block BP-811587157-172.17.0.2-1733449972741:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:07,922 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:32910 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:32769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:32910 dst: /127.0.0.1:32769
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:07,922 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43824 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:35333:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43824 dst: /127.0.0.1:35333
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:07,923 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@63e7f9e9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:07,924 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3cbdbe64{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:07,924 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:07,924 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@782c3e10{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:07,924 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6413b81{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:07,929 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:07,929 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:07,929 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-811587157-172.17.0.2-1733449972741 (Datanode Uuid e5478376-a8e8-4192-a861-8710614e826e) service to localhost/127.0.0.1:41913
2024-12-06T01:53:07,929 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:07,930 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data7/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:07,930 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data8/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:07,930 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:09,005 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:09,675 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:09,919 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:09,920 WARN  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]
2024-12-06T01:53:09,920 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C45229%2C1733449973578:(num 1733449985904) roll requested
2024-12-06T01:53:09,921 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.1733449989920
2024-12-06T01:53:09,925 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43828 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data6]'}, localName='127.0.0.1:35333', datanodeUuid='8d884b35-0de4-4814-a84f-1b0422766da0', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741840_1022 to mirror 127.0.0.1:37205
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:09,925 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37205
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:09,925 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43828 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T01:53:09,925 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:09,925 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741840_1022
2024-12-06T01:53:09,925 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43828 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:35333:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43828 dst: /127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:09,926 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:09,926 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 after 4007ms
2024-12-06T01:53:09,927 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:09,928 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:09,928 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741841_1023
2024-12-06T01:53:09,928 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:09,932 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32769
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:09,932 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43434 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741842_1024] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741842_1024 to mirror 127.0.0.1:32769
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:09,932 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:09,932 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741842_1024
2024-12-06T01:53:09,932 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43434 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741842_1024] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T01:53:09,933 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43434 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43434 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:09,933 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:09,935 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-06T01:53:09,938 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:09,938 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:09,938 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:09,938 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:09,938 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:09,939 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449985904 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449989920
2024-12-06T01:53:09,940 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39777:39777),(127.0.0.1/127.0.0.1:33985:33985)]
2024-12-06T01:53:09,940 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:09,940 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449985904 is not closed yet, will try archiving it next time
2024-12-06T01:53:09,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35333 is added to blk_1073741839_1021 (size=3600)
2024-12-06T01:53:10,342 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:11,005 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741839_1021 (size=3600)
2024-12-06T01:53:11,675 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,939 WARN  [ResponseProcessor for block BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1025
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,940 WARN  [DataStreamer for file /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449989920 block BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:11,940 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,940 WARN  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]
2024-12-06T01:53:11,940 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C45229%2C1733449973578:(num 1733449989920) roll requested
2024-12-06T01:53:11,940 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43836 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:35333:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43836 dst: /127.0.0.1:35333
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:11,940 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43440 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43440 dst: /127.0.0.1:44497
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:11,941 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.1733449991940
2024-12-06T01:53:11,945 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@621d6abe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:11,945 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@511ebf5d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:11,945 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:11,946 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a3456d5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:11,946 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b3a85a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:11,948 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:11,948 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-811587157-172.17.0.2-1733449972741 (Datanode Uuid 8d884b35-0de4-4814-a84f-1b0422766da0) service to localhost/127.0.0.1:41913
2024-12-06T01:53:11,948 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:11,948 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:11,949 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data5/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:11,949 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data6/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:11,949 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:11,951 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,951 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:11,951 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741844_1027
2024-12-06T01:53:11,952 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:11,954 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,954 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:11,954 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741845_1028
2024-12-06T01:53:11,955 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45229 {}] regionserver.HRegion(8855): Flush requested on ce716e29eafea16423da07d2956326df
2024-12-06T01:53:11,962 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce716e29eafea16423da07d2956326df 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:53:11,965 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32769
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,965 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43464 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741846_1029 to mirror 127.0.0.1:32769
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:11,965 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:11,966 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741846_1029
2024-12-06T01:53:11,966 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43464 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T01:53:11,966 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43464 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43464 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:11,966 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:11,969 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,969 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:11,969 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741847_1030
2024-12-06T01:53:11,970 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:11,970 WARN  [IPC Server handler 3 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:11,971 WARN  [IPC Server handler 3 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:11,971 WARN  [IPC Server handler 3 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:11,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/fc087672b09540a998141e3dd33d4bb3 is 1080, key is row0002/info:/1733449987932/Put/seqid=0
2024-12-06T01:53:11,992 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,992 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:11,992 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741849_1032
2024-12-06T01:53:11,993 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:11,995 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:11,995 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:11,995 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741850_1033
2024-12-06T01:53:11,999 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:12,005 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:12,005 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:12,005 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:12,005 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:12,006 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:12,006 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449989920 with entries=12, filesize=12.96 KB; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449991940
2024-12-06T01:53:12,006 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37205
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:12,006 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:12,006 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741851_1034
2024-12-06T01:53:12,007 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:12,006 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43486 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741851_1034 to mirror 127.0.0.1:37205
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:12,007 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43486 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:12,008 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:43486 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43486 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:12,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741843_1026 (size=13274)
2024-12-06T01:53:12,009 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:12,009 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:12,009 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741852_1035
2024-12-06T01:53:12,010 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:12,011 WARN  [IPC Server handler 2 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:12,011 WARN  [IPC Server handler 2 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:12,011 WARN  [IPC Server handler 2 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:12,027 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33985:33985)]
2024-12-06T01:53:12,028 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:12,028 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449989920 is not closed yet, will try archiving it next time
2024-12-06T01:53:12,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741853_1036 (size=10347)
2024-12-06T01:53:12,032 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/fc087672b09540a998141e3dd33d4bb3
2024-12-06T01:53:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/fc087672b09540a998141e3dd33d4bb3 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/fc087672b09540a998141e3dd33d4bb3
2024-12-06T01:53:12,066 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/fc087672b09540a998141e3dd33d4bb3, entries=5, sequenceid=11, filesize=10.1 K
2024-12-06T01:53:12,067 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=7.35 KB/7531 for ce716e29eafea16423da07d2956326df in 105ms, sequenceid=11, compaction requested=false
2024-12-06T01:53:12,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45229 {}] regionserver.HRegion(8855): Flush requested on ce716e29eafea16423da07d2956326df
2024-12-06T01:53:12,190 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce716e29eafea16423da07d2956326df 1/1 column families, dataSize=8.40 KB heapSize=9.25 KB
2024-12-06T01:53:12,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/258f87a2c8d349e782ad991ce9278ec5 is 1080, key is row0007/info:/1733449991963/Put/seqid=0
2024-12-06T01:53:12,197 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:12,197 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:12,197 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741854_1037
2024-12-06T01:53:12,198 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:12,200 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:12,201 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:12,201 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741855_1038
2024-12-06T01:53:12,202 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:12,203 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:12,203 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:12,204 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741856_1039
2024-12-06T01:53:12,204 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:12,206 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:12,206 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:12,206 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741857_1040
2024-12-06T01:53:12,207 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:12,208 WARN  [IPC Server handler 4 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:12,208 WARN  [IPC Server handler 4 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:12,208 WARN  [IPC Server handler 4 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:12,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741858_1041 (size=12506)
2024-12-06T01:53:12,409 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:12,410 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449985904 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs/00c764f7c3aa%2C45229%2C1733449973578.1733449985904
2024-12-06T01:53:12,619 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.40 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/258f87a2c8d349e782ad991ce9278ec5
2024-12-06T01:53:12,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/258f87a2c8d349e782ad991ce9278ec5 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5
2024-12-06T01:53:12,632 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5, entries=7, sequenceid=22, filesize=12.2 K
2024-12-06T01:53:12,633 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.40 KB/8606, heapSize ~9.23 KB/9456, currentSize=2.10 KB/2150 for ce716e29eafea16423da07d2956326df in 444ms, sequenceid=22, compaction requested=false
2024-12-06T01:53:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:12,634 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-06T01:53:12,634 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:12,634 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5 because midkey is the same as first or last row
2024-12-06T01:53:13,006 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45229 {}] regionserver.HRegion(8855): Flush requested on ce716e29eafea16423da07d2956326df
2024-12-06T01:53:13,610 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce716e29eafea16423da07d2956326df 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T01:53:13,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/8b32129010fd4ac398a5455cdf1f197a is 1079, key is tmprow/info:/1733449993608/Put/seqid=0
2024-12-06T01:53:13,620 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:13,620 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:13,620 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741859_1042
2024-12-06T01:53:13,621 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:13,624 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:13,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56466 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741860_1043 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:13,624 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:13,624 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741860_1043
2024-12-06T01:53:13,624 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56466 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:13,625 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56466 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56466 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:13,625 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:13,626 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:13,627 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:13,627 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741861_1044
2024-12-06T01:53:13,627 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:13,630 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37205
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:13,630 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56482 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741862_1045 to mirror 127.0.0.1:37205
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:13,630 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:13,631 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741862_1045
2024-12-06T01:53:13,631 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56482 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:13,631 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:13,631 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56482 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56482 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:13,632 WARN  [IPC Server handler 4 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:13,632 WARN  [IPC Server handler 4 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:13,632 WARN  [IPC Server handler 4 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:13,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741863_1046 (size=6027)
2024-12-06T01:53:13,675 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,028 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,029 WARN  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]
2024-12-06T01:53:14,029 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C45229%2C1733449973578:(num 1733449991940) roll requested
2024-12-06T01:53:14,029 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.1733449994029
2024-12-06T01:53:14,033 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,034 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:14,034 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741864_1047
2024-12-06T01:53:14,034 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:14,036 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,037 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:14,037 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741865_1048
2024-12-06T01:53:14,038 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:14,038 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/8b32129010fd4ac398a5455cdf1f197a
2024-12-06T01:53:14,041 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32769
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56494 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741866_1049 to mirror 127.0.0.1:32769
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,041 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:14,041 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741866_1049
2024-12-06T01:53:14,041 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56494 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T01:53:14,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56494 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56494 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,042 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:14,043 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,043 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:14,044 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741867_1050
2024-12-06T01:53:14,044 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:14,045 WARN  [IPC Server handler 2 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:14,045 WARN  [IPC Server handler 2 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:14,045 WARN  [IPC Server handler 2 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:14,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/8b32129010fd4ac398a5455cdf1f197a as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/8b32129010fd4ac398a5455cdf1f197a
2024-12-06T01:53:14,056 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/8b32129010fd4ac398a5455cdf1f197a, entries=1, sequenceid=32, filesize=5.9 K
2024-12-06T01:53:14,058 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for ce716e29eafea16423da07d2956326df in 447ms, sequenceid=32, compaction requested=true
2024-12-06T01:53:14,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:14,058 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-06T01:53:14,058 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:14,058 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5 because midkey is the same as first or last row
2024-12-06T01:53:14,060 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:14,060 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:14,060 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:14,060 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:14,060 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:14,061 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449991940 with entries=15, filesize=13.17 KB; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449994029
2024-12-06T01:53:14,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce716e29eafea16423da07d2956326df:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:53:14,061 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:53:14,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:53:14,063 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741848_1031 (size=13490)
2024-12-06T01:53:14,063 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:53:14,064 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HStore(1541): ce716e29eafea16423da07d2956326df/info is initiating minor compaction (all files)
2024-12-06T01:53:14,064 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce716e29eafea16423da07d2956326df/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:14,064 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/fc087672b09540a998141e3dd33d4bb3, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/8b32129010fd4ac398a5455cdf1f197a] into tmpdir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp, totalSize=28.2 K
2024-12-06T01:53:14,064 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33985:33985)]
2024-12-06T01:53:14,064 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:14,064 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449991940 is not closed yet, will try archiving it next time
2024-12-06T01:53:14,064 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc087672b09540a998141e3dd33d4bb3, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733449987932
2024-12-06T01:53:14,064 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449989920 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs/00c764f7c3aa%2C45229%2C1733449973578.1733449989920
2024-12-06T01:53:14,065 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258f87a2c8d349e782ad991ce9278ec5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=22, earliestPutTs=1733449991963
2024-12-06T01:53:14,065 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b32129010fd4ac398a5455cdf1f197a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733449993608
2024-12-06T01:53:14,085 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce716e29eafea16423da07d2956326df#info#compaction#21 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:53:14,085 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/33fd632cd7d04fce869773882803053e is 1080, key is row0002/info:/1733449987932/Put/seqid=0
2024-12-06T01:53:14,087 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,087 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:14,087 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741869_1052
2024-12-06T01:53:14,088 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:14,090 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45875
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56526 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741870_1053 to mirror 127.0.0.1:45875
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,090 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:14,090 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741870_1053
2024-12-06T01:53:14,090 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56526 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:14,091 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56526 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56526 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,091 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:14,093 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,093 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:14,093 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741871_1054
2024-12-06T01:53:14,100 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:14,102 WARN  [Thread-954 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:14,102 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56534 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741872_1055] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741872_1055 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,103 WARN  [Thread-954 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:14,103 WARN  [Thread-954 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741872_1055
2024-12-06T01:53:14,103 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56534 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741872_1055] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:14,103 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56534 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741872_1055] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56534 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,103 WARN  [Thread-954 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:14,104 WARN  [IPC Server handler 0 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:14,104 WARN  [IPC Server handler 0 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:14,104 WARN  [IPC Server handler 0 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:14,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741873_1056 (size=17994)
2024-12-06T01:53:14,119 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/33fd632cd7d04fce869773882803053e as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e
2024-12-06T01:53:14,134 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce716e29eafea16423da07d2956326df/info of ce716e29eafea16423da07d2956326df into 33fd632cd7d04fce869773882803053e(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:53:14,134 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:14,134 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df., storeName=ce716e29eafea16423da07d2956326df/info, priority=13, startTime=1733449994058; duration=0sec
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e because midkey is the same as first or last row
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e because midkey is the same as first or last row
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-06T01:53:14,135 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:14,136 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e because midkey is the same as first or last row
2024-12-06T01:53:14,136 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:53:14,136 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce716e29eafea16423da07d2956326df:info
2024-12-06T01:53:14,464 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 is not closed yet, will try archiving it next time
2024-12-06T01:53:14,779 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@13f7448a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741843_1026 to 127.0.0.1:32769 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:14,780 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17c8c22c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741853_1036 to 127.0.0.1:32769 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:15,006 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45229 {}] regionserver.HRegion(8855): Flush requested on ce716e29eafea16423da07d2956326df
2024-12-06T01:53:15,037 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce716e29eafea16423da07d2956326df 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T01:53:15,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/d29c5da969cb4bc2b9952887b6489d09 is 1079, key is tmprow/info:/1733449995035/Put/seqid=0
2024-12-06T01:53:15,045 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:15,045 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK], DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]) is bad.
2024-12-06T01:53:15,045 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741874_1057
2024-12-06T01:53:15,046 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32769,DS-2af26174-4be5-43c1-a63a-7d2eb15097cf,DISK]
2024-12-06T01:53:15,048 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:15,048 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]) is bad.
2024-12-06T01:53:15,048 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741875_1058
2024-12-06T01:53:15,048 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45875,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]
2024-12-06T01:53:15,050 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:15,050 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:15,050 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741876_1059
2024-12-06T01:53:15,050 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:15,052 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:15,052 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:15,052 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741877_1060
2024-12-06T01:53:15,053 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:15,053 WARN  [IPC Server handler 3 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-06T01:53:15,053 WARN  [IPC Server handler 3 on default port 41913 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-06T01:53:15,054 WARN  [IPC Server handler 3 on default port 41913 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-06T01:53:15,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741878_1061 (size=6027)
2024-12-06T01:53:15,460 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/d29c5da969cb4bc2b9952887b6489d09
2024-12-06T01:53:15,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/d29c5da969cb4bc2b9952887b6489d09 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d29c5da969cb4bc2b9952887b6489d09
2024-12-06T01:53:15,474 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d29c5da969cb4bc2b9952887b6489d09, entries=1, sequenceid=43, filesize=5.9 K
2024-12-06T01:53:15,475 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for ce716e29eafea16423da07d2956326df in 438ms, sequenceid=43, compaction requested=false
2024-12-06T01:53:15,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:15,476 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-06T01:53:15,476 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:15,476 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e because midkey is the same as first or last row
2024-12-06T01:53:15,676 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:15,779 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@13f7448a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741863_1046 to 127.0.0.1:35333 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:15,779 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@17c8c22c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741858_1041 to 127.0.0.1:45875 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:16,065 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:16,065 WARN  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-06T01:53:16,266 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:16,271 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:16,272 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:16,272 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:16,272 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:53:16,275 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23601ca6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:16,275 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f39d55e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:16,404 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d723f63{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/java.io.tmpdir/jetty-localhost-46587-hadoop-hdfs-3_4_1-tests_jar-_-any-5714862981126243084/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:16,405 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2537b75{HTTP/1.1, (http/1.1)}{localhost:46587}
2024-12-06T01:53:16,405 INFO  [Time-limited test {}] server.Server(415): Started @129673ms
2024-12-06T01:53:16,406 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:16,520 WARN  [Thread-981 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:16,530 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe815ab5ec13d1e20 with lease ID 0x951377814a1ffe72: from storage DS-9b14a558-ddd4-4393-8642-eb18a93bb91e node DatanodeRegistration(127.0.0.1:40235, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=43275, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:53:16,530 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe815ab5ec13d1e20 with lease ID 0x951377814a1ffe72: from storage DS-b4f93e80-73e0-43f3-9b97-a2100a700915 node DatanodeRegistration(127.0.0.1:40235, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=43275, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:17,007 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:17,676 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:17,779 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@13f7448a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741873_1056 to 127.0.0.1:32769 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:17,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741848_1031 (size=13490)
2024-12-06T01:53:18,065 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:18,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741878_1061 (size=6027)
2024-12-06T01:53:19,007 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:19,677 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:20,066 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:21,007 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:21,677 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:22,066 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:23,008 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:23,509 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:53:23,678 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,066 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,187 ERROR [FSHLog-0-hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData-prefix:00c764f7c3aa,36647,1733449973529 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,187 WARN  [FSHLog-0-hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData-prefix:00c764f7c3aa,36647,1733449973529 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,187 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C36647%2C1733449973529:(num 1733449974106) roll requested
2024-12-06T01:53:24,188 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C36647%2C1733449973529.1733450004187
2024-12-06T01:53:24,192 WARN  [Thread-1003 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,192 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-819121948_22 at /127.0.0.1:60234 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data4]'}, localName='127.0.0.1:40235', datanodeUuid='feb48222-10f2-4cb1-95f9-4dc13a60c6e4', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741879_1062 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:24,192 WARN  [Thread-1003 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40235,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:24,193 WARN  [Thread-1003 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741879_1062
2024-12-06T01:53:24,193 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-819121948_22 at /127.0.0.1:60234 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T01:53:24,193 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-819121948_22 at /127.0.0.1:60234 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:40235:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60234 dst: /127.0.0.1:40235
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:24,193 WARN  [Thread-1003 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:24,194 WARN  [Thread-1003 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,194 WARN  [Thread-1003 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK], DatanodeInfoWithStorage[127.0.0.1:40235,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]) is bad.
2024-12-06T01:53:24,195 WARN  [Thread-1003 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741880_1063
2024-12-06T01:53:24,195 WARN  [Thread-1003 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]
2024-12-06T01:53:24,199 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:24,199 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:24,199 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:24,199 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:24,199 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:24,200 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 with entries=54, filesize=26.65 KB; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733450004187
2024-12-06T01:53:24,200 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,200 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:24,200 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106
2024-12-06T01:53:24,201 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33985:33985),(127.0.0.1/127.0.0.1:43275:43275)]
2024-12-06T01:53:24,201 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 is not closed yet, will try archiving it next time
2024-12-06T01:53:24,201 WARN  [IPC Server handler 2 on default port 41913 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 has not been closed. Lease recovery is in progress. RecoveryId = 1065 for block blk_1073741830_1006
2024-12-06T01:53:24,201 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 after 1ms
2024-12-06T01:53:25,008 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:26,067 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:26,544 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6494e58d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:37205,null,null])
java.net.ConnectException: Call From 00c764f7c3aa/172.17.0.2 to localhost:37741 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-06T01:53:26,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741833_1020 (size=455)
2024-12-06T01:53:26,945 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449974421 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs/00c764f7c3aa%2C45229%2C1733449973578.1733449974421
2024-12-06T01:53:26,946 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449991940 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs/00c764f7c3aa%2C45229%2C1733449973578.1733449991940
2024-12-06T01:53:27,009 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:28,067 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:28,203 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/WALs/00c764f7c3aa,36647,1733449973529/00c764f7c3aa%2C36647%2C1733449973529.1733449974106 after 4002ms
2024-12-06T01:53:29,009 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:29,528 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@47c0273a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40235, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=43275, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741833_1020 to 127.0.0.1:35333 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:29,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741835_1011 (size=393)
2024-12-06T01:53:30,068 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:30,527 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@47c0273a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40235, datanodeUuid=feb48222-10f2-4cb1-95f9-4dc13a60c6e4, infoPort=43275, infoSecurePort=0, ipcPort=34973, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741829_1005 to 127.0.0.1:35333 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:30,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:53:31,009 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:31,470 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.1733450011470
2024-12-06T01:53:31,478 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:31,478 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:31,478 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:31,478 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:31,478 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:31,479 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449994029 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733450011470
2024-12-06T01:53:31,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43275:43275),(127.0.0.1/127.0.0.1:33985:33985)]
2024-12-06T01:53:31,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.1733449994029 is not closed yet, will try archiving it next time
2024-12-06T01:53:31,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741868_1051 (size=12911)
2024-12-06T01:53:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45229 {}] regionserver.HRegion(8855): Flush requested on ce716e29eafea16423da07d2956326df
2024-12-06T01:53:31,485 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce716e29eafea16423da07d2956326df 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-06T01:53:31,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/84891e0ea1af4b749997d8444ce849ef is 1080, key is row0013/info:/1733450011482/Put/seqid=0
2024-12-06T01:53:31,494 WARN  [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:31,494 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56938 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741883_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741883_1067 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:31,494 WARN  [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741883_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:31,494 WARN  [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741883_1067
2024-12-06T01:53:31,494 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56938 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741883_1067] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:31,494 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:56938 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741883_1067] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56938 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:31,495 WARN  [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:31,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741884_1068 (size=8190)
2024-12-06T01:53:31,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741884_1068 (size=8190)
2024-12-06T01:53:31,911 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/84891e0ea1af4b749997d8444ce849ef
2024-12-06T01:53:31,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/84891e0ea1af4b749997d8444ce849ef as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/84891e0ea1af4b749997d8444ce849ef
2024-12-06T01:53:31,925 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/84891e0ea1af4b749997d8444ce849ef, entries=3, sequenceid=53, filesize=8.0 K
2024-12-06T01:53:31,926 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11833 for ce716e29eafea16423da07d2956326df in 441ms, sequenceid=53, compaction requested=true
2024-12-06T01:53:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=31.5 K, sizeToCheck=16.0 K
2024-12-06T01:53:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e because midkey is the same as first or last row
2024-12-06T01:53:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce716e29eafea16423da07d2956326df:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:53:31,926 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:53:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:53:31,928 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 32211 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:53:31,928 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HStore(1541): ce716e29eafea16423da07d2956326df/info is initiating minor compaction (all files)
2024-12-06T01:53:31,928 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce716e29eafea16423da07d2956326df/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:31,928 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d29c5da969cb4bc2b9952887b6489d09, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/84891e0ea1af4b749997d8444ce849ef] into tmpdir=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp, totalSize=31.5 K
2024-12-06T01:53:31,929 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33fd632cd7d04fce869773882803053e, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1733449987932
2024-12-06T01:53:31,929 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.Compactor(225): Compacting d29c5da969cb4bc2b9952887b6489d09, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733449995035
2024-12-06T01:53:31,930 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84891e0ea1af4b749997d8444ce849ef, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733449996053
2024-12-06T01:53:31,948 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce716e29eafea16423da07d2956326df#info#compaction#24 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:53:31,949 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/0866e513fcb749eb84d4ca1781288608 is 1080, key is row0002/info:/1733449987932/Put/seqid=0
2024-12-06T01:53:31,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741885_1069 (size=20267)
2024-12-06T01:53:31,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741885_1069 (size=20267)
2024-12-06T01:53:31,968 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/0866e513fcb749eb84d4ca1781288608 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/0866e513fcb749eb84d4ca1781288608
2024-12-06T01:53:31,977 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce716e29eafea16423da07d2956326df/info of ce716e29eafea16423da07d2956326df into 0866e513fcb749eb84d4ca1781288608(size=19.8 K), total size for store is 19.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:53:31,977 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce716e29eafea16423da07d2956326df:

2024-12-06T01:53:31,977 INFO  [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df., storeName=ce716e29eafea16423da07d2956326df/info, priority=13, startTime=1733450011926; duration=0sec
2024-12-06T01:53:31,977 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=19.8 K, sizeToCheck=16.0 K
2024-12-06T01:53:31,977 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:31,977 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/0866e513fcb749eb84d4ca1781288608 because midkey is the same as first or last row
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=19.8 K, sizeToCheck=16.0 K
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/0866e513fcb749eb84d4ca1781288608 because midkey is the same as first or last row
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=19.8 K, sizeToCheck=16.0 K
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/0866e513fcb749eb84d4ca1781288608 because midkey is the same as first or last row
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:53:31,978 DEBUG [RS:0;00c764f7c3aa:45229-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce716e29eafea16423da07d2956326df:info
2024-12-06T01:53:32,068 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,068 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-06T01:53:32,110 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:53:32,111 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:53:32,111 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:53:32,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:32,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:32,111 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:53:32,112 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:53:32,112 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=924258338, stopped=false
2024-12-06T01:53:32,112 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,36647,1733449973529
2024-12-06T01:53:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:53:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:53:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:53:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:32,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:53:32,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:53:32,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:53:32,120 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:53:32,120 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:53:32,120 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:53:32,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:32,121 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,45229,1733449973578' *****
2024-12-06T01:53:32,121 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:53:32,121 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,37779,1733449974953' *****
2024-12-06T01:53:32,121 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:53:32,121 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:53:32,121 INFO  [RS:1;00c764f7c3aa:37779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:53:32,121 INFO  [RS:1;00c764f7c3aa:37779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:53:32,121 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,37779,1733449974953
2024-12-06T01:53:32,121 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:53:32,122 INFO  [RS:1;00c764f7c3aa:37779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;00c764f7c3aa:37779.
2024-12-06T01:53:32,122 DEBUG [RS:1;00c764f7c3aa:37779 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:53:32,122 DEBUG [RS:1;00c764f7c3aa:37779 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:32,122 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,37779,1733449974953; all regions closed.
2024-12-06T01:53:32,122 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:53:32,123 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:53:32,123 INFO  [RS:0;00c764f7c3aa:45229 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:53:32,123 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:53:32,123 INFO  [RS:0;00c764f7c3aa:45229 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:53:32,123 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(3091): Received CLOSE for ce716e29eafea16423da07d2956326df
2024-12-06T01:53:32,124 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,124 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,124 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,124 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,124 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,45229,1733449973578
2024-12-06T01:53:32,124 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:53:32,124 INFO  [RS:0;00c764f7c3aa:45229 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:45229.
2024-12-06T01:53:32,125 DEBUG [RS:0;00c764f7c3aa:45229 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:53:32,125 DEBUG [RS:0;00c764f7c3aa:45229 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:32,125 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:53:32,125 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:53:32,125 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:53:32,125 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:53:32,125 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,125 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ce716e29eafea16423da07d2956326df, disabling compactions & flushes
2024-12-06T01:53:32,125 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:32,127 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:32,127 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df. after waiting 0 ms
2024-12-06T01:53:32,127 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:32,127 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ce716e29eafea16423da07d2956326df 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-06T01:53:32,129 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T01:53:32,129 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1325): Online Regions={ce716e29eafea16423da07d2956326df=TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T01:53:32,129 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ce716e29eafea16423da07d2956326df
2024-12-06T01:53:32,130 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:53:32,130 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:53:32,130 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:53:32,130 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:53:32,130 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:53:32,130 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-06T01:53:32,130 ERROR [FSHLog-0-hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c-prefix:00c764f7c3aa,45229,1733449973578.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,131 WARN  [FSHLog-0-hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c-prefix:00c764f7c3aa,45229,1733449973578.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,131 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C45229%2C1733449973578.meta:.meta(num 1733449974801) roll requested
2024-12-06T01:53:32,131 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45229%2C1733449973578.meta.1733450012131.meta
2024-12-06T01:53:32,142 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/d707e431d9c448fa83a751bb26c8f614 is 1080, key is row0015/info:/1733450011486/Put/seqid=0
2024-12-06T01:53:32,148 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,148 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,148 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
2024-12-06T01:53:32,149 WARN  [IPC Server handler 0 on default port 41913 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741837_1013
2024-12-06T01:53:32,149 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140 after 1ms
2024-12-06T01:53:32,150 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:60274 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741886_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data4]'}, localName='127.0.0.1:40235', datanodeUuid='feb48222-10f2-4cb1-95f9-4dc13a60c6e4', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741886_1070 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,150 WARN  [Thread-1040 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1070
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,151 WARN  [Thread-1040 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741886_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40235,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:32,151 WARN  [Thread-1040 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741886_1070
2024-12-06T01:53:32,151 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:60274 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741886_1070] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-06T01:53:32,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:60274 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741886_1070] {}] datanode.DataXceiver(331): 127.0.0.1:40235:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60274 dst: /127.0.0.1:40235
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,152 WARN  [Thread-1041 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1071
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,152 WARN  [Thread-1041 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741887_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40235,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:32,152 WARN  [Thread-1041 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741887_1071
2024-12-06T01:53:32,153 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:60288 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741887_1071] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data4]'}, localName='127.0.0.1:40235', datanodeUuid='feb48222-10f2-4cb1-95f9-4dc13a60c6e4', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741887_1071 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,154 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:60288 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741887_1071] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:32,154 WARN  [Thread-1040 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:32,154 WARN  [Thread-1041 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:32,155 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:60288 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741887_1071] {}] datanode.DataXceiver(331): 127.0.0.1:40235:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60288 dst: /127.0.0.1:40235
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,185 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,185 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,185 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,186 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,186 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,186 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733450012131.meta
2024-12-06T01:53:32,191 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,191 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37205,DS-aadc3028-d916-4a7c-ab66-8bdc97980a4e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741889_1074 (size=14660)
2024-12-06T01:53:32,192 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
2024-12-06T01:53:32,193 WARN  [IPC Server handler 1 on default port 41913 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741834_1010
2024-12-06T01:53:32,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741889_1074 (size=14660)
2024-12-06T01:53:32,194 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta after 2ms
2024-12-06T01:53:32,194 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/d707e431d9c448fa83a751bb26c8f614
2024-12-06T01:53:32,204 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/.tmp/info/d707e431d9c448fa83a751bb26c8f614 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d707e431d9c448fa83a751bb26c8f614
2024-12-06T01:53:32,218 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d707e431d9c448fa83a751bb26c8f614, entries=9, sequenceid=68, filesize=14.3 K
2024-12-06T01:53:32,220 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11833, heapSize ~12.61 KB/12912, currentSize=0 B/0 for ce716e29eafea16423da07d2956326df in 92ms, sequenceid=68, compaction requested=false
2024-12-06T01:53:32,220 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33985:33985),(127.0.0.1/127.0.0.1:43275:43275)]
2024-12-06T01:53:32,220 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta is not closed yet, will try archiving it next time
2024-12-06T01:53:32,233 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/fc087672b09540a998141e3dd33d4bb3, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/8b32129010fd4ac398a5455cdf1f197a, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d29c5da969cb4bc2b9952887b6489d09, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/84891e0ea1af4b749997d8444ce849ef] to archive
2024-12-06T01:53:32,235 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T01:53:32,240 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/fc087672b09540a998141e3dd33d4bb3 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/fc087672b09540a998141e3dd33d4bb3
2024-12-06T01:53:32,242 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/258f87a2c8d349e782ad991ce9278ec5
2024-12-06T01:53:32,244 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/33fd632cd7d04fce869773882803053e
2024-12-06T01:53:32,246 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/8b32129010fd4ac398a5455cdf1f197a to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/8b32129010fd4ac398a5455cdf1f197a
2024-12-06T01:53:32,249 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d29c5da969cb4bc2b9952887b6489d09 to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/d29c5da969cb4bc2b9952887b6489d09
2024-12-06T01:53:32,250 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:53:32,251 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/84891e0ea1af4b749997d8444ce849ef to hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/info/84891e0ea1af4b749997d8444ce849ef
2024-12-06T01:53:32,252 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=00c764f7c3aa:36647 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-06T01:53:32,252 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [fc087672b09540a998141e3dd33d4bb3=10347, 258f87a2c8d349e782ad991ce9278ec5=12506, 33fd632cd7d04fce869773882803053e=17994, 8b32129010fd4ac398a5455cdf1f197a=6027, d29c5da969cb4bc2b9952887b6489d09=6027, 84891e0ea1af4b749997d8444ce849ef=8190]
2024-12-06T01:53:32,273 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/info/b0f161d8c2e44183b9bb10ebdda142ec is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df./info:regioninfo/1733449975411/Put/seqid=0
2024-12-06T01:53:32,286 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T01:53:32,286 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T01:53:32,287 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:57004 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741890_1076] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10]'}, localName='127.0.0.1:44497', datanodeUuid='5c7b67fe-46ca-4ff3-b4e3-df0458697a40', xmitsInProgress=0}:Exception transferring block BP-811587157-172.17.0.2-1733449972741:blk_1073741890_1076 to mirror 127.0.0.1:35333
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,287 WARN  [Thread-1054 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1076
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35333
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,288 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:57004 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741890_1076] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-06T01:53:32,288 WARN  [Thread-1054 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741890_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44497,DS-70b3dc6e-f084-47a8-923c-3de73a3ce0e1,DISK], DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:32,288 WARN  [Thread-1054 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741890_1076
2024-12-06T01:53:32,288 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-221302824_22 at /127.0.0.1:57004 [Receiving block BP-811587157-172.17.0.2-1733449972741:blk_1073741890_1076] {}] datanode.DataXceiver(331): 127.0.0.1:44497:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57004 dst: /127.0.0.1:44497
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,289 WARN  [Thread-1054 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:32,291 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ce716e29eafea16423da07d2956326df/recovered.edits/71.seqid, newMaxSeqId=71, maxSeqId=1
2024-12-06T01:53:32,292 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:32,292 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ce716e29eafea16423da07d2956326df:
Waiting for close lock at 1733450012125Running coprocessor pre-close hooks at 1733450012125Disabling compacts and flushes for region at 1733450012125Disabling writes for close at 1733450012127 (+2 ms)Obtaining lock to block concurrent updates at 1733450012127Preparing flush snapshotting stores in ce716e29eafea16423da07d2956326df at 1733450012127Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df., syncing WAL and waiting on mvcc, flushsize=dataSize=11833, getHeapSize=12912, getOffHeapSize=0, getCellsCount=11 at 1733450012127Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df. at 1733450012129 (+2 ms)Flushing ce716e29eafea16423da07d2956326df/info: creating writer at 1733450012129Flushing ce716e29eafea16423da07d2956326df/info: appending metadata at 1733450012141 (+12 ms)Flushing ce716e29eafea16423da07d2956326df/info: closing flushed file at 1733450012141Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@328073f0: reopening flushed file at 1733450012203 (+62 ms)Finished flush of dataSize ~11.56 KB/11833, heapSize ~12.61 KB/12912, currentSize=0 B/0 for ce716e29eafea16423da07d2956326df in 92ms, sequenceid=68, compaction requested=false at 1733450012220 (+17 ms)Writing region close event to WAL at 1733450012276 (+56 ms)Running coprocessor post-close hooks at 1733450012292 (+16 ms)Closed at 1733450012292
2024-12-06T01:53:32,292 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733449975040.ce716e29eafea16423da07d2956326df.
2024-12-06T01:53:32,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741891_1077 (size=7089)
2024-12-06T01:53:32,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741891_1077 (size=7089)
2024-12-06T01:53:32,330 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:53:32,530 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:53:32,720 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/info/b0f161d8c2e44183b9bb10ebdda142ec
2024-12-06T01:53:32,730 DEBUG [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:53:32,761 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/ns/56c69fb2a91745b68456426195f01118 is 43, key is default/ns:d/1733449974849/Put/seqid=0
2024-12-06T01:53:32,764 WARN  [Thread-1063 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,764 WARN  [Thread-1063 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741892_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:40235,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:32,764 WARN  [Thread-1063 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741892_1078
2024-12-06T01:53:32,765 WARN  [Thread-1063 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:32,778 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@13f7448a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44497, datanodeUuid=5c7b67fe-46ca-4ff3-b4e3-df0458697a40, infoPort=33985, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=742622661;c=1733449972741):Failed to transfer BP-811587157-172.17.0.2-1733449972741:blk_1073741868_1051 to 127.0.0.1:35333 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:32,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741893_1079 (size=5153)
2024-12-06T01:53:32,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741893_1079 (size=5153)
2024-12-06T01:53:32,781 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/ns/56c69fb2a91745b68456426195f01118
2024-12-06T01:53:32,815 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/table/d24459f8ec4148988404d6c71211e109 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733449975423/Put/seqid=0
2024-12-06T01:53:32,818 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1080
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:32,819 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-811587157-172.17.0.2-1733449972741:blk_1073741894_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK], DatanodeInfoWithStorage[127.0.0.1:40235,DS-9b14a558-ddd4-4393-8642-eb18a93bb91e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]) is bad.
2024-12-06T01:53:32,819 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-811587157-172.17.0.2-1733449972741:blk_1073741894_1080
2024-12-06T01:53:32,820 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35333,DS-41b9c483-c038-49a2-a0d2-2491f9bbc9d2,DISK]
2024-12-06T01:53:32,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741895_1081 (size=5424)
2024-12-06T01:53:32,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741895_1081 (size=5424)
2024-12-06T01:53:32,834 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/table/d24459f8ec4148988404d6c71211e109
2024-12-06T01:53:32,843 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/info/b0f161d8c2e44183b9bb10ebdda142ec as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/info/b0f161d8c2e44183b9bb10ebdda142ec
2024-12-06T01:53:32,858 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/info/b0f161d8c2e44183b9bb10ebdda142ec, entries=10, sequenceid=11, filesize=6.9 K
2024-12-06T01:53:32,859 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/ns/56c69fb2a91745b68456426195f01118 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/ns/56c69fb2a91745b68456426195f01118
2024-12-06T01:53:32,867 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/ns/56c69fb2a91745b68456426195f01118, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T01:53:32,870 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/.tmp/table/d24459f8ec4148988404d6c71211e109 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/table/d24459f8ec4148988404d6c71211e109
2024-12-06T01:53:32,879 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/table/d24459f8ec4148988404d6c71211e109, entries=2, sequenceid=11, filesize=5.3 K
2024-12-06T01:53:32,880 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 750ms, sequenceid=11, compaction requested=false
2024-12-06T01:53:32,892 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T01:53:32,893 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:53:32,893 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:53:32,894 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450012129Running coprocessor pre-close hooks at 1733450012129Disabling compacts and flushes for region at 1733450012129Disabling writes for close at 1733450012130 (+1 ms)Obtaining lock to block concurrent updates at 1733450012130Preparing flush snapshotting stores in 1588230740 at 1733450012130Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733450012131 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733450012222 (+91 ms)Flushing 1588230740/info: creating writer at 1733450012222Flushing 1588230740/info: appending metadata at 1733450012271 (+49 ms)Flushing 1588230740/info: closing flushed file at 1733450012272 (+1 ms)Flushing 1588230740/ns: creating writer at 1733450012732 (+460 ms)Flushing 1588230740/ns: appending metadata at 1733450012760 (+28 ms)Flushing 1588230740/ns: closing flushed file at 1733450012760Flushing 1588230740/table: creating writer at 1733450012789 (+29 ms)Flushing 1588230740/table: appending metadata at 1733450012814 (+25 ms)Flushing 1588230740/table: closing flushed file at 1733450012814Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4927d7b9: reopening flushed file at 1733450012842 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b1bfad: reopening flushed file at 1733450012858 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d7d14c: reopening flushed file at 1733450012868 (+10 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 750ms, sequenceid=11, compaction requested=false at 1733450012880 (+12 ms)Writing region close event to WAL at 1733450012887 (+7 ms)Running coprocessor post-close hooks at 1733450012893 (+6 ms)Closed at 1733450012893
2024-12-06T01:53:32,894 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:53:32,931 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,45229,1733449973578; all regions closed.
2024-12-06T01:53:32,931 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,932 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,932 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,932 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,934 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:32,937 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741888_1073 (size=825)
2024-12-06T01:53:32,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741888_1073 (size=825)
2024-12-06T01:53:33,006 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:53:33,007 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T01:53:33,007 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T01:53:33,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-06T01:53:33,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:53:33,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:53:33,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:53:33,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741836_1012 (size=76)
2024-12-06T01:53:34,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:53:34,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:53:34,866 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T01:53:34,866 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T01:53:35,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:53:35,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:53:35,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741839_1021 (size=3600)
2024-12-06T01:53:36,150 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140 after 4002ms
2024-12-06T01:53:36,195 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta after 4003ms
2024-12-06T01:53:36,548 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3f04fee8 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-811587157-172.17.0.2-1733449972741:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:37205,null,null])
java.net.ConnectException: Call From 00c764f7c3aa/172.17.0.2 to localhost:37741 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-06T01:53:37,148 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-06T01:53:37,150 DEBUG [RS:1;00c764f7c3aa:37779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs
2024-12-06T01:53:37,150 INFO  [RS:1;00c764f7c3aa:37779 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C37779%2C1733449974953:(num 1733449975140)
2024-12-06T01:53:37,150 DEBUG [RS:1;00c764f7c3aa:37779 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:37,150 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:53:37,150 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:53:37,151 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-06T01:53:37,151 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:53:37,151 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:53:37,151 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:53:37,151 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:53:37,151 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:53:37,151 INFO  [RS:1;00c764f7c3aa:37779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37779
2024-12-06T01:53:37,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:53:37,154 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:53:37,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,37779,1733449974953
2024-12-06T01:53:37,155 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,37779,1733449974953]
2024-12-06T01:53:37,156 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:37,157 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,37779,1733449974953 already deleted, retry=false
2024-12-06T01:53:37,157 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,37779,1733449974953 expired; onlineServers=1
2024-12-06T01:53:37,255 INFO  [RS:1;00c764f7c3aa:37779 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:53:37,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:53:37,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37779-0x100650737ee0002, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:53:37,256 INFO  [RS:1;00c764f7c3aa:37779 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,37779,1733449974953; zookeeper connection closed.
2024-12-06T01:53:37,256 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@30ad593d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@30ad593d
2024-12-06T01:53:37,293 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,320 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,320 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,824 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:53:37,842 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,844 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,844 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,845 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:37,935 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-06T01:53:37,938 DEBUG [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs
2024-12-06T01:53:37,938 INFO  [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C45229%2C1733449973578.meta:.meta(num 1733450012131)
2024-12-06T01:53:37,939 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:37,939 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:37,939 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:37,939 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:37,940 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:37,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741882_1066 (size=17476)
2024-12-06T01:53:37,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741882_1066 (size=17476)
2024-12-06T01:53:38,157 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:38,197 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:38,221 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-12-06T01:53:38,347 DEBUG [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/oldWALs
2024-12-06T01:53:38,347 INFO  [RS:0;00c764f7c3aa:45229 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C45229%2C1733449973578:(num 1733450011470)
2024-12-06T01:53:38,347 DEBUG [RS:0;00c764f7c3aa:45229 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:38,347 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:53:38,347 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:53:38,347 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-06T01:53:38,347 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:53:38,347 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:53:38,348 INFO  [RS:0;00c764f7c3aa:45229 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45229
2024-12-06T01:53:38,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,45229,1733449973578
2024-12-06T01:53:38,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:53:38,351 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:53:38,352 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,45229,1733449973578]
2024-12-06T01:53:38,354 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,45229,1733449973578 already deleted, retry=false
2024-12-06T01:53:38,354 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,45229,1733449973578 expired; onlineServers=0
2024-12-06T01:53:38,354 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,36647,1733449973529' *****
2024-12-06T01:53:38,354 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:53:38,354 INFO  [M:0;00c764f7c3aa:36647 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:53:38,354 INFO  [M:0;00c764f7c3aa:36647 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:53:38,355 DEBUG [M:0;00c764f7c3aa:36647 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:53:38,355 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:53:38,355 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449974188 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449974188,5,FailOnTimeoutGroup]
2024-12-06T01:53:38,355 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449974187 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449974187,5,FailOnTimeoutGroup]
2024-12-06T01:53:38,355 DEBUG [M:0;00c764f7c3aa:36647 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:53:38,355 INFO  [M:0;00c764f7c3aa:36647 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:53:38,355 INFO  [M:0;00c764f7c3aa:36647 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:53:38,355 DEBUG [M:0;00c764f7c3aa:36647 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:53:38,355 INFO  [M:0;00c764f7c3aa:36647 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:53:38,355 INFO  [M:0;00c764f7c3aa:36647 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:53:38,356 INFO  [M:0;00c764f7c3aa:36647 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:53:38,356 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:53:38,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:53:38,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:38,358 DEBUG [M:0;00c764f7c3aa:36647 {}] zookeeper.ZKUtil(347): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:53:38,358 WARN  [M:0;00c764f7c3aa:36647 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:53:38,359 INFO  [M:0;00c764f7c3aa:36647 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/.lastflushedseqids
2024-12-06T01:53:38,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741896_1082 (size=130)
2024-12-06T01:53:38,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741896_1082 (size=130)
2024-12-06T01:53:38,367 INFO  [M:0;00c764f7c3aa:36647 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:53:38,367 INFO  [M:0;00c764f7c3aa:36647 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:53:38,367 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:53:38,367 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:38,368 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:38,368 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:53:38,368 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:38,368 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.24 KB heapSize=29.47 KB
2024-12-06T01:53:38,386 DEBUG [M:0;00c764f7c3aa:36647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cf934d65c134804be44bca364d4d1a8 is 82, key is hbase:meta,,1/info:regioninfo/1733449974831/Put/seqid=0
2024-12-06T01:53:38,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741897_1083 (size=5672)
2024-12-06T01:53:38,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741897_1083 (size=5672)
2024-12-06T01:53:38,399 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cf934d65c134804be44bca364d4d1a8
2024-12-06T01:53:38,436 DEBUG [M:0;00c764f7c3aa:36647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c26a89f45163426ea402b4ed58e5f1f7 is 773, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733449975428/Put/seqid=0
2024-12-06T01:53:38,444 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741898_1084 (size=6254)
2024-12-06T01:53:38,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741898_1084 (size=6254)
2024-12-06T01:53:38,445 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c26a89f45163426ea402b4ed58e5f1f7
2024-12-06T01:53:38,450 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c26a89f45163426ea402b4ed58e5f1f7
2024-12-06T01:53:38,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:53:38,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45229-0x100650737ee0001, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:53:38,453 INFO  [RS:0;00c764f7c3aa:45229 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:53:38,453 INFO  [RS:0;00c764f7c3aa:45229 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,45229,1733449973578; zookeeper connection closed.
2024-12-06T01:53:38,453 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@34e3029c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@34e3029c
2024-12-06T01:53:38,453 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-06T01:53:38,467 DEBUG [M:0;00c764f7c3aa:36647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c66e8a2078404f5eb3407e4a9c627d4a is 69, key is 00c764f7c3aa,37779,1733449974953/rs:state/1733449974991/Put/seqid=0
2024-12-06T01:53:38,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741899_1085 (size=5224)
2024-12-06T01:53:38,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741899_1085 (size=5224)
2024-12-06T01:53:38,483 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c66e8a2078404f5eb3407e4a9c627d4a
2024-12-06T01:53:38,511 DEBUG [M:0;00c764f7c3aa:36647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/53435413fe844d1081db6eb4ff8a625d is 52, key is load_balancer_on/state:d/1733449974936/Put/seqid=0
2024-12-06T01:53:38,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741900_1086 (size=5056)
2024-12-06T01:53:38,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741900_1086 (size=5056)
2024-12-06T01:53:38,518 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/53435413fe844d1081db6eb4ff8a625d
2024-12-06T01:53:38,526 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cf934d65c134804be44bca364d4d1a8 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cf934d65c134804be44bca364d4d1a8
2024-12-06T01:53:38,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741830_1065 (size=27294)
2024-12-06T01:53:38,535 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cf934d65c134804be44bca364d4d1a8, entries=8, sequenceid=60, filesize=5.5 K
2024-12-06T01:53:38,536 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c26a89f45163426ea402b4ed58e5f1f7 as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c26a89f45163426ea402b4ed58e5f1f7
2024-12-06T01:53:38,543 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c26a89f45163426ea402b4ed58e5f1f7
2024-12-06T01:53:38,543 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c26a89f45163426ea402b4ed58e5f1f7, entries=6, sequenceid=60, filesize=6.1 K
2024-12-06T01:53:38,544 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c66e8a2078404f5eb3407e4a9c627d4a as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c66e8a2078404f5eb3407e4a9c627d4a
2024-12-06T01:53:38,550 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c66e8a2078404f5eb3407e4a9c627d4a, entries=2, sequenceid=60, filesize=5.1 K
2024-12-06T01:53:38,551 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/53435413fe844d1081db6eb4ff8a625d as hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/53435413fe844d1081db6eb4ff8a625d
2024-12-06T01:53:38,557 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/53435413fe844d1081db6eb4ff8a625d, entries=1, sequenceid=60, filesize=4.9 K
2024-12-06T01:53:38,558 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 190ms, sequenceid=60, compaction requested=false
2024-12-06T01:53:38,560 INFO  [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:38,560 DEBUG [M:0;00c764f7c3aa:36647 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450018367Disabling compacts and flushes for region at 1733450018367Disabling writes for close at 1733450018368 (+1 ms)Obtaining lock to block concurrent updates at 1733450018368Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733450018368Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23793, getHeapSize=30112, getOffHeapSize=0, getCellsCount=71 at 1733450018368Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733450018369 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733450018369Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733450018385 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733450018386 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733450018420 (+34 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733450018436 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733450018436Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733450018451 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733450018467 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733450018467Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733450018490 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733450018510 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733450018510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35d5024f: reopening flushed file at 1733450018524 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@213da61: reopening flushed file at 1733450018535 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34e556f0: reopening flushed file at 1733450018543 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d976c38: reopening flushed file at 1733450018550 (+7 ms)Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 190ms, sequenceid=60, compaction requested=false at 1733450018558 (+8 ms)Writing region close event to WAL at 1733450018560 (+2 ms)Closed at 1733450018560
2024-12-06T01:53:38,560 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:38,561 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:38,561 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:38,561 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:38,561 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:38,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40235 is added to blk_1073741881_1064 (size=1045)
2024-12-06T01:53:38,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44497 is added to blk_1073741881_1064 (size=1045)
2024-12-06T01:53:38,564 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:53:38,564 INFO  [M:0;00c764f7c3aa:36647 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:53:38,564 INFO  [M:0;00c764f7c3aa:36647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36647
2024-12-06T01:53:38,564 INFO  [M:0;00c764f7c3aa:36647 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:53:38,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:53:38,667 INFO  [M:0;00c764f7c3aa:36647 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:53:38,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36647-0x100650737ee0000, quorum=127.0.0.1:53502, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:53:38,670 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d723f63{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:38,671 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2537b75{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:38,671 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:38,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f39d55e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:38,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23601ca6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:38,673 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:38,672 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1cd04703 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:37205,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:37741 , LocalHost:localPort 00c764f7c3aa/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-06T01:53:38,673 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-811587157-172.17.0.2-1733449972741 (Datanode Uuid feb48222-10f2-4cb1-95f9-4dc13a60c6e4) service to localhost/127.0.0.1:41913
2024-12-06T01:53:38,673 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:38,673 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:38,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data3/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:38,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data4/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:38,674 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1cd04703 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(293): Failed to updateBlock (newblock=BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1072, datanode=DatanodeInfoWithStorage[127.0.0.1:40235,null,null])
org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException: Replica not found for BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013[numBytes=85,originalReplicaState=RWR]. The block may have been removed recently by the balancer or by intentionally reducing the replication factor. This condition is usually harmless. To be certain, please check the preceding datanode log messages for signs of a more serious issue.
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.updateReplicaUnderRecovery(FsDatasetImpl.java:3104) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode.updateReplicaUnderRecovery(DataNode.java:3537) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$BlockRecord.updateReplicaUnderRecovery(BlockRecoveryWorker.java:88) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$BlockRecord.access$700(BlockRecoveryWorker.java:71) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.syncBlock(BlockRecoveryWorker.java:289) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:183) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:38,674 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1cd04703 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:40235,null,null], DatanodeInfoWithStorage[127.0.0.1:37205,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: Cannot recover BP-811587157-172.17.0.2-1733449972741:blk_1073741837_1013, the following datanodes failed: [DatanodeInfoWithStorage[127.0.0.1:40235,null,null]]
2024-12-06T01:53:38,674 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1cd04703 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37205,null,null])
java.io.IOException: No block pool offer service for bpid=BP-811587157-172.17.0.2-1733449972741
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:38,674 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1cd04703 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:40235,null,null])
java.io.IOException: No block pool offer service for bpid=BP-811587157-172.17.0.2-1733449972741
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:38,675 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1cd04703 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37205,null,null], DatanodeInfoWithStorage[127.0.0.1:40235,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-811587157-172.17.0.2-1733449972741:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:37205,null,null], DatanodeInfoWithStorage[127.0.0.1:40235,null,null]]
2024-12-06T01:53:38,675 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:38,678 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d93430d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:38,678 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@241fb63b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:38,678 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:38,678 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47917ab3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:38,679 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@90666a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:38,680 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:38,681 WARN  [BP-811587157-172.17.0.2-1733449972741 heartbeating to localhost/127.0.0.1:41913 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-811587157-172.17.0.2-1733449972741 (Datanode Uuid 5c7b67fe-46ca-4ff3-b4e3-df0458697a40) service to localhost/127.0.0.1:41913
2024-12-06T01:53:38,681 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data9/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:38,682 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/cluster_0e0af9de-57cc-e65b-7230-b87f7cc1b294/data/data10/current/BP-811587157-172.17.0.2-1733449972741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:38,682 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:38,682 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:38,682 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:38,692 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70e75057{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:53:38,693 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a936a30{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:38,693 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:38,693 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e8e5e50{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:38,693 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9fcb45e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:38,703 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:53:38,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:53:38,768 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 79)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41913
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:41913 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:39051
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:41913
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$895/0x00007f7ad0befa48.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41913
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:41913 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41913
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$895/0x00007f7ad0befa48.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:41913 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:39051
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41913
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:41913
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41913
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:41913 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=454 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 289) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9882 (was 9865) - AvailableMemoryMB LEAK? -
2024-12-06T01:53:38,776 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=9881
2024-12-06T01:53:38,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:53:38,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.log.dir so I do NOT create it in target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de
2024-12-06T01:53:38,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d72cc0c-c1c0-51cf-ca84-e24eb7c34f84/hadoop.tmp.dir so I do NOT create it in target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833, deleteOnExit=true
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/test.cache.data in system properties and HBase conf
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:53:38,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:53:38,778 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:53:38,778 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:53:38,778 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:53:38,778 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:53:38,778 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:53:38,778 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:53:38,779 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:53:38,794 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:53:38,866 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:38,871 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:38,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:38,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:38,872 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:53:38,875 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:38,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e676e1c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:38,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2eab1cda{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:38,995 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c4c555c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-45691-hadoop-hdfs-3_4_1-tests_jar-_-any-9205719061120915896/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:53:38,996 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30337913{HTTP/1.1, (http/1.1)}{localhost:45691}
2024-12-06T01:53:38,996 INFO  [Time-limited test {}] server.Server(415): Started @152263ms
2024-12-06T01:53:39,010 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:53:39,090 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:39,094 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:39,097 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:39,097 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:39,097 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:53:39,098 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18af7984{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:39,099 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53068b03{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:39,158 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:39,198 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:39,239 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c33f78d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-44483-hadoop-hdfs-3_4_1-tests_jar-_-any-4037360766468218173/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:39,239 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68d35ecb{HTTP/1.1, (http/1.1)}{localhost:44483}
2024-12-06T01:53:39,240 INFO  [Time-limited test {}] server.Server(415): Started @152508ms
2024-12-06T01:53:39,241 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:39,288 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:39,292 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:39,293 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:39,294 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:39,294 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:53:39,295 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d796cca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:39,295 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20237619{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:39,354 WARN  [Thread-1192 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data1/current/BP-2122666517-172.17.0.2-1733450018813/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:39,355 WARN  [Thread-1193 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data2/current/BP-2122666517-172.17.0.2-1733450018813/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:39,389 WARN  [Thread-1171 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:39,392 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3214dd6720b38623 with lease ID 0x2a11476020524baa: Processing first storage report for DS-802f1003-ee70-40b6-8e4f-0698a5ca818b from datanode DatanodeRegistration(127.0.0.1:39943, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=35715, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813)
2024-12-06T01:53:39,392 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3214dd6720b38623 with lease ID 0x2a11476020524baa: from storage DS-802f1003-ee70-40b6-8e4f-0698a5ca818b node DatanodeRegistration(127.0.0.1:39943, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=35715, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:39,393 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3214dd6720b38623 with lease ID 0x2a11476020524baa: Processing first storage report for DS-f27dfa02-04a0-47d7-b7e9-1f9d2985ff17 from datanode DatanodeRegistration(127.0.0.1:39943, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=35715, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813)
2024-12-06T01:53:39,393 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3214dd6720b38623 with lease ID 0x2a11476020524baa: from storage DS-f27dfa02-04a0-47d7-b7e9-1f9d2985ff17 node DatanodeRegistration(127.0.0.1:39943, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=35715, infoSecurePort=0, ipcPort=38629, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:39,433 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49046baa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-46469-hadoop-hdfs-3_4_1-tests_jar-_-any-1586939220461799424/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:39,434 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b18e233{HTTP/1.1, (http/1.1)}{localhost:46469}
2024-12-06T01:53:39,434 INFO  [Time-limited test {}] server.Server(415): Started @152702ms
2024-12-06T01:53:39,435 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:39,550 WARN  [Thread-1218 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data3/current/BP-2122666517-172.17.0.2-1733450018813/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:39,551 WARN  [Thread-1219 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data4/current/BP-2122666517-172.17.0.2-1733450018813/current, will proceed with Du for space computation calculation, 
2024-12-06T01:53:39,577 WARN  [Thread-1207 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:39,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a3ce2d45e4ad15b with lease ID 0x2a11476020524bab: Processing first storage report for DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c from datanode DatanodeRegistration(127.0.0.1:39441, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=34327, infoSecurePort=0, ipcPort=44681, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813)
2024-12-06T01:53:39,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a3ce2d45e4ad15b with lease ID 0x2a11476020524bab: from storage DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c node DatanodeRegistration(127.0.0.1:39441, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=34327, infoSecurePort=0, ipcPort=44681, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:39,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a3ce2d45e4ad15b with lease ID 0x2a11476020524bab: Processing first storage report for DS-62337f26-659f-494e-9516-552e9c59eb05 from datanode DatanodeRegistration(127.0.0.1:39441, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=34327, infoSecurePort=0, ipcPort=44681, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813)
2024-12-06T01:53:39,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a3ce2d45e4ad15b with lease ID 0x2a11476020524bab: from storage DS-62337f26-659f-494e-9516-552e9c59eb05 node DatanodeRegistration(127.0.0.1:39441, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=34327, infoSecurePort=0, ipcPort=44681, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:39,592 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de
2024-12-06T01:53:39,595 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/zookeeper_0, clientPort=54846, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:53:39,596 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54846
2024-12-06T01:53:39,597 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,598 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:53:39,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:53:39,608 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7 with version=8
2024-12-06T01:53:39,608 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:53:39,610 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:53:39,610 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:53:39,610 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:53:39,611 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:53:39,611 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:53:39,611 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:53:39,611 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:53:39,611 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:53:39,611 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33927
2024-12-06T01:53:39,613 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33927 connecting to ZooKeeper ensemble=127.0.0.1:54846
2024-12-06T01:53:39,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339270x0, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:53:39,620 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33927-0x1006507ebf10000 connected
2024-12-06T01:53:39,639 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,641 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,643 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:53:39,644 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7, hbase.cluster.distributed=false
2024-12-06T01:53:39,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:53:39,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33927
2024-12-06T01:53:39,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33927
2024-12-06T01:53:39,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33927
2024-12-06T01:53:39,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33927
2024-12-06T01:53:39,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33927
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:53:39,664 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:53:39,665 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41247
2024-12-06T01:53:39,666 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41247 connecting to ZooKeeper ensemble=127.0.0.1:54846
2024-12-06T01:53:39,667 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,668 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412470x0, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:53:39,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412470x0, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:53:39,673 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:53:39,673 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41247-0x1006507ebf10001 connected
2024-12-06T01:53:39,674 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:53:39,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:53:39,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:53:39,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41247
2024-12-06T01:53:39,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41247
2024-12-06T01:53:39,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41247
2024-12-06T01:53:39,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41247
2024-12-06T01:53:39,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41247
2024-12-06T01:53:39,703 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:33927
2024-12-06T01:53:39,703 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:39,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:53:39,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:53:39,706 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:39,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:53:39,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:39,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:39,709 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:53:39,709 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,33927,1733450019610 from backup master directory
2024-12-06T01:53:39,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:53:39,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:39,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:53:39,712 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:53:39,712 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:39,721 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/hbase.id] with ID: 6b34fcba-167a-4cde-9527-54f344f08887
2024-12-06T01:53:39,721 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/.tmp/hbase.id
2024-12-06T01:53:39,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:53:39,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:53:39,734 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/.tmp/hbase.id]:[hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/hbase.id]
2024-12-06T01:53:39,748 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:39,748 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:53:39,749 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T01:53:39,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:39,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:39,761 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:53:39,761 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:53:39,762 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:53:39,763 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:53:39,764 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:53:39,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:53:39,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:53:40,159 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:40,173 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store
2024-12-06T01:53:40,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:53:40,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:53:40,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:53:40,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:53:40,184 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:40,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:40,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:53:40,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:40,184 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:53:40,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450020184Disabling compacts and flushes for region at 1733450020184Disabling writes for close at 1733450020184Writing region close event to WAL at 1733450020184Closed at 1733450020184
2024-12-06T01:53:40,185 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/.initializing
2024-12-06T01:53:40,185 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:40,188 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C33927%2C1733450019610, suffix=, logDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610, archiveDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/oldWALs, maxLogs=10
2024-12-06T01:53:40,188 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C33927%2C1733450019610.1733450020188
2024-12-06T01:53:40,199 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:40,200 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188
2024-12-06T01:53:40,201 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35715:35715),(127.0.0.1/127.0.0.1:34327:34327)]
2024-12-06T01:53:40,204 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:53:40,204 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:53:40,204 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,204 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,208 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:53:40,210 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:53:40,210 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,211 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,211 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:53:40,212 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:53:40,212 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,213 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:53:40,213 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:53:40,214 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:53:40,214 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,215 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:53:40,215 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:53:40,216 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:53:40,216 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,217 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:53:40,217 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,218 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,218 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,219 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,219 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,220 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:53:40,221 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:53:40,224 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:53:40,224 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743434, jitterRate=-0.0546751469373703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:53:40,226 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733450020204Initializing all the Stores at 1733450020205 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020205Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450020208 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450020208Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450020208Cleaning up temporary data from old regions at 1733450020219 (+11 ms)Region opened successfully at 1733450020225 (+6 ms)
2024-12-06T01:53:40,228 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:53:40,232 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391114dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:53:40,233 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:53:40,233 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:53:40,233 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:53:40,233 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:53:40,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T01:53:40,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T01:53:40,234 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:53:40,237 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:53:40,238 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:53:40,240 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:53:40,240 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:53:40,241 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:53:40,242 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:53:40,243 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:53:40,244 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:53:40,245 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:53:40,247 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:53:40,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:53:40,250 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:53:40,252 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:53:40,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:53:40,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:53:40,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,254 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,33927,1733450019610, sessionid=0x1006507ebf10000, setting cluster-up flag (Was=false)
2024-12-06T01:53:40,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,264 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:53:40,266 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:40,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,282 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:53:40,284 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:40,289 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:53:40,291 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:53:40,291 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:53:40,292 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:53:40,292 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,33927,1733450019610
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:53:40,293 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:53:40,293 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:53:40,293 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:53:40,293 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:53:40,294 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:53:40,294 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,294 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:53:40,294 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,304 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733450050304
2024-12-06T01:53:40,305 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:53:40,305 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:53:40,305 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:53:40,305 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:53:40,305 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:53:40,305 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:53:40,306 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:53:40,306 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:53:40,307 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,307 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:53:40,308 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,308 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:53:40,308 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:53:40,308 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:53:40,309 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:53:40,309 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:53:40,312 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450020309,5,FailOnTimeoutGroup]
2024-12-06T01:53:40,315 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450020312,5,FailOnTimeoutGroup]
2024-12-06T01:53:40,315 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,315 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:53:40,315 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,315 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:53:40,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:53:40,323 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:53:40,323 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7
2024-12-06T01:53:40,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:53:40,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:53:40,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:53:40,336 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:53:40,338 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:53:40,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:53:40,341 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:53:40,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:53:40,343 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:53:40,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,343 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,343 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:53:40,345 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:53:40,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,345 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:53:40,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740
2024-12-06T01:53:40,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740
2024-12-06T01:53:40,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:53:40,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:53:40,348 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:53:40,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:53:40,352 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:53:40,353 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=694331, jitterRate=-0.1171128898859024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:53:40,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733450020335Initializing all the Stores at 1733450020336 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020336Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020336Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450020336Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020336Cleaning up temporary data from old regions at 1733450020348 (+12 ms)Region opened successfully at 1733450020353 (+5 ms)
2024-12-06T01:53:40,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:53:40,354 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:53:40,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:53:40,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:53:40,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:53:40,354 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:53:40,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450020354Disabling compacts and flushes for region at 1733450020354Disabling writes for close at 1733450020354Writing region close event to WAL at 1733450020354Closed at 1733450020354
2024-12-06T01:53:40,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:53:40,356 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:53:40,356 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:53:40,358 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:53:40,359 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:53:40,387 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(746): ClusterId : 6b34fcba-167a-4cde-9527-54f344f08887
2024-12-06T01:53:40,387 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:53:40,390 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:53:40,390 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:53:40,393 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:53:40,393 DEBUG [RS:0;00c764f7c3aa:41247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2887615f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:53:40,411 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:41247
2024-12-06T01:53:40,411 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:53:40,411 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:53:40,411 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:53:40,412 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,33927,1733450019610 with port=41247, startcode=1733450019663
2024-12-06T01:53:40,412 DEBUG [RS:0;00c764f7c3aa:41247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:53:40,415 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53441, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:53:40,415 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33927 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,416 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33927 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,418 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7
2024-12-06T01:53:40,418 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39825
2024-12-06T01:53:40,418 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:53:40,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:53:40,421 DEBUG [RS:0;00c764f7c3aa:41247 {}] zookeeper.ZKUtil(111): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,421 WARN  [RS:0;00c764f7c3aa:41247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:53:40,421 INFO  [RS:0;00c764f7c3aa:41247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:53:40,421 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,421 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,41247,1733450019663]
2024-12-06T01:53:40,424 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:53:40,426 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:53:40,428 INFO  [RS:0;00c764f7c3aa:41247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:53:40,428 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,428 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:53:40,429 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:53:40,429 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,429 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,429 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,429 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:53:40,430 DEBUG [RS:0;00c764f7c3aa:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:53:40,432 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,432 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,432 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,432 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,432 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,433 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,41247,1733450019663-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:53:40,455 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:53:40,455 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,41247,1733450019663-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,455 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,455 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.Replication(171): 00c764f7c3aa,41247,1733450019663 started
2024-12-06T01:53:40,471 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,471 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,41247,1733450019663, RpcServer on 00c764f7c3aa/172.17.0.2:41247, sessionid=0x1006507ebf10001
2024-12-06T01:53:40,472 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:53:40,472 DEBUG [RS:0;00c764f7c3aa:41247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,472 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,41247,1733450019663'
2024-12-06T01:53:40,472 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:53:40,472 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,41247,1733450019663'
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:53:40,473 DEBUG [RS:0;00c764f7c3aa:41247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:53:40,473 INFO  [RS:0;00c764f7c3aa:41247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:53:40,474 INFO  [RS:0;00c764f7c3aa:41247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:53:40,510 WARN  [00c764f7c3aa:33927 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:53:40,576 INFO  [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C41247%2C1733450019663, suffix=, logDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663, archiveDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs, maxLogs=32
2024-12-06T01:53:40,577 INFO  [RS:0;00c764f7c3aa:41247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:53:40,583 INFO  [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:53:40,584 DEBUG [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35715:35715),(127.0.0.1/127.0.0.1:34327:34327)]
2024-12-06T01:53:40,760 DEBUG [00c764f7c3aa:33927 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:53:40,761 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,762 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,41247,1733450019663, state=OPENING
2024-12-06T01:53:40,764 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:53:40,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:53:40,767 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:53:40,767 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:53:40,767 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:53:40,767 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,41247,1733450019663}]
2024-12-06T01:53:40,921 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:53:40,923 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48739, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:53:40,926 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:53:40,927 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:53:40,928 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C41247%2C1733450019663.meta, suffix=.meta, logDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663, archiveDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs, maxLogs=32
2024-12-06T01:53:40,929 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta
2024-12-06T01:53:40,934 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta
2024-12-06T01:53:40,935 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35715:35715),(127.0.0.1/127.0.0.1:34327:34327)]
2024-12-06T01:53:40,936 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:53:40,936 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:53:40,936 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:53:40,936 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:53:40,937 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:53:40,937 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:53:40,937 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:53:40,937 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:53:40,938 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:53:40,939 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:53:40,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,940 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,940 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:53:40,941 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:53:40,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:53:40,942 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:53:40,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:53:40,943 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:53:40,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:40,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:53:40,944 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:53:40,945 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740
2024-12-06T01:53:40,946 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740
2024-12-06T01:53:40,947 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:53:40,947 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:53:40,948 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:53:40,949 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:53:40,950 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=846349, jitterRate=0.07618880271911621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:53:40,950 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:53:40,951 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733450020937Writing region info on filesystem at 1733450020937Initializing all the Stores at 1733450020938 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020938Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020938Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450020938Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450020938Cleaning up temporary data from old regions at 1733450020947 (+9 ms)Running coprocessor post-open hooks at 1733450020950 (+3 ms)Region opened successfully at 1733450020951 (+1 ms)
2024-12-06T01:53:40,952 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733450020920
2024-12-06T01:53:40,955 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:53:40,955 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:53:40,956 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,957 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,41247,1733450019663, state=OPEN
2024-12-06T01:53:40,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:53:40,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:53:40,967 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:40,967 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:53:40,967 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:53:40,970 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:53:40,971 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,41247,1733450019663 in 200 msec
2024-12-06T01:53:40,973 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:53:40,974 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 615 msec
2024-12-06T01:53:40,974 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:53:40,974 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:53:40,976 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:53:40,976 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,41247,1733450019663, seqNum=-1]
2024-12-06T01:53:40,976 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:53:40,978 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:53:40,983 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 692 msec
2024-12-06T01:53:40,983 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733450020983, completionTime=-1
2024-12-06T01:53:40,983 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:53:40,983 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:53:40,985 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:53:40,985 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733450080985
2024-12-06T01:53:40,985 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733450140985
2024-12-06T01:53:40,985 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T01:53:40,986 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33927,1733450019610-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,986 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33927,1733450019610-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,986 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33927,1733450019610-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,986 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:33927, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,986 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,986 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:40,988 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.278sec
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33927,1733450019610-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:53:40,990 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33927,1733450019610-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:53:40,993 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:53:40,993 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:53:40,993 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,33927,1733450019610-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:53:41,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bc2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:53:41,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,33927,-1 for getting cluster id
2024-12-06T01:53:41,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:53:41,091 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b34fcba-167a-4cde-9527-54f344f08887'
2024-12-06T01:53:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:53:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b34fcba-167a-4cde-9527-54f344f08887"

2024-12-06T01:53:41,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c9d755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:53:41,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,33927,-1]
2024-12-06T01:53:41,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:53:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:53:41,098 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:53:41,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf8e6ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:53:41,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:53:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,41247,1733450019663, seqNum=-1]
2024-12-06T01:53:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:53:41,103 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:53:41,105 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:41,105 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:53:41,108 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:53:41,108 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-06T01:53:41,108 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-06T01:53:41,108 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T01:53:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,33927,1733450019610
2024-12-06T01:53:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b4b96b6
2024-12-06T01:53:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T01:53:41,111 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T01:53:41,112 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T01:53:41,112 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T01:53:41,112 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:53:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T01:53:41,115 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T01:53:41,115 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:41,115 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-06T01:53:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:53:41,116 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T01:53:41,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741835_1011 (size=395)
2024-12-06T01:53:41,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741835_1011 (size=395)
2024-12-06T01:53:41,127 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0b85de0754ff399666bd5af46ffbcf80, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7
2024-12-06T01:53:41,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39441 is added to blk_1073741836_1012 (size=78)
2024-12-06T01:53:41,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39943 is added to blk_1073741836_1012 (size=78)
2024-12-06T01:53:41,142 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:53:41,142 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 0b85de0754ff399666bd5af46ffbcf80, disabling compactions & flushes
2024-12-06T01:53:41,142 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,142 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,142 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80. after waiting 0 ms
2024-12-06T01:53:41,142 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,142 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,142 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0b85de0754ff399666bd5af46ffbcf80:
Waiting for close lock at 1733450021142Disabling compacts and flushes for region at 1733450021142Disabling writes for close at 1733450021142Writing region close event to WAL at 1733450021142Closed at 1733450021142
2024-12-06T01:53:41,144 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T01:53:41,144 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733450021144"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733450021144"}]},"ts":"1733450021144"}
2024-12-06T01:53:41,146 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T01:53:41,148 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T01:53:41,148 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733450021148"}]},"ts":"1733450021148"}
2024-12-06T01:53:41,150 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-06T01:53:41,150 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0b85de0754ff399666bd5af46ffbcf80, ASSIGN}]
2024-12-06T01:53:41,151 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0b85de0754ff399666bd5af46ffbcf80, ASSIGN
2024-12-06T01:53:41,152 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0b85de0754ff399666bd5af46ffbcf80, ASSIGN; state=OFFLINE, location=00c764f7c3aa,41247,1733450019663; forceNewPlan=false, retain=false
2024-12-06T01:53:41,159 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:41,199 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:41,303 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0b85de0754ff399666bd5af46ffbcf80, regionState=OPENING, regionLocation=00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:41,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0b85de0754ff399666bd5af46ffbcf80, ASSIGN because future has completed
2024-12-06T01:53:41,307 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b85de0754ff399666bd5af46ffbcf80, server=00c764f7c3aa,41247,1733450019663}]
2024-12-06T01:53:41,465 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,466 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b85de0754ff399666bd5af46ffbcf80, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:53:41,466 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,466 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:53:41,466 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,466 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,468 INFO  [StoreOpener-0b85de0754ff399666bd5af46ffbcf80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0b85de0754ff399666bd5af46ffbcf80 
2024-12-06T01:53:41,470 INFO  [StoreOpener-0b85de0754ff399666bd5af46ffbcf80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b85de0754ff399666bd5af46ffbcf80 columnFamilyName info
2024-12-06T01:53:41,470 DEBUG [StoreOpener-0b85de0754ff399666bd5af46ffbcf80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:53:41,470 INFO  [StoreOpener-0b85de0754ff399666bd5af46ffbcf80-1 {}] regionserver.HStore(327): Store=0b85de0754ff399666bd5af46ffbcf80/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:53:41,471 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,471 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,472 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,472 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,472 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,475 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,481 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:53:41,481 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0b85de0754ff399666bd5af46ffbcf80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735108, jitterRate=-0.06526236236095428}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:53:41,482 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:53:41,483 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0b85de0754ff399666bd5af46ffbcf80:
Running coprocessor pre-open hook at 1733450021467Writing region info on filesystem at 1733450021467Initializing all the Stores at 1733450021468 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450021468Cleaning up temporary data from old regions at 1733450021472 (+4 ms)Running coprocessor post-open hooks at 1733450021482 (+10 ms)Region opened successfully at 1733450021483 (+1 ms)
2024-12-06T01:53:41,484 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80., pid=6, masterSystemTime=1733450021460
2024-12-06T01:53:41,487 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,487 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:41,488 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0b85de0754ff399666bd5af46ffbcf80, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,41247,1733450019663
2024-12-06T01:53:41,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b85de0754ff399666bd5af46ffbcf80, server=00c764f7c3aa,41247,1733450019663 because future has completed
2024-12-06T01:53:41,496 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T01:53:41,496 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b85de0754ff399666bd5af46ffbcf80, server=00c764f7c3aa,41247,1733450019663 in 186 msec
2024-12-06T01:53:41,499 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T01:53:41,500 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0b85de0754ff399666bd5af46ffbcf80, ASSIGN in 346 msec
2024-12-06T01:53:41,501 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T01:53:41,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733450021501"}]},"ts":"1733450021501"}
2024-12-06T01:53:41,504 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-06T01:53:41,505 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T01:53:41,508 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 393 msec
2024-12-06T01:53:42,160 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:42,200 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:43,161 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:43,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:43,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:53:43,405 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T01:53:43,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T01:53:43,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-06T01:53:43,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:53:43,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T01:53:44,161 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:44,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:45,162 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:45,202 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:46,163 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:46,203 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:46,439 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:53:46,453 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,454 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,454 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,454 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,455 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,455 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,462 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:53:46,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T01:53:46,476 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-06T01:53:47,164 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:47,203 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:48,164 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:48,204 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:49,165 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:49,205 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:50,166 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:50,205 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:51,166 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:53:51,200 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-06T01:53:51,200 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-06T01:53:51,203 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T01:53:51,203 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:53:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80., hostname=00c764f7c3aa,41247,1733450019663, seqNum=2]
2024-12-06T01:53:51,206 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:52,167 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:52,207 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:53,167 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:53,207 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:53,209 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:53:53,210 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:53,211 WARN  [DataStreamer for file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 block BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK], DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]) is bad.
2024-12-06T01:53:53,211 WARN  [PacketResponder: BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39441] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,211 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:53,211 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:59264 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39943:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59264 dst: /127.0.0.1:39943
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,212 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:53,212 WARN  [DataStreamer for file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 block BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK], DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]) is bad.
2024-12-06T01:53:53,212 WARN  [DataStreamer for file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta block BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK], DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39441,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]) is bad.
2024-12-06T01:53:53,212 WARN  [PacketResponder: BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39441] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,212 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:33562 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39441:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33562 dst: /127.0.0.1:39441
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,212 WARN  [PacketResponder: BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39441] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2015138003_22 at /127.0.0.1:59242 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39943:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59242 dst: /127.0.0.1:39943
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:59274 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39943:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59274 dst: /127.0.0.1:39943
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:33572 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39441:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33572 dst: /127.0.0.1:39441
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2015138003_22 at /127.0.0.1:33550 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39441:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33550 dst: /127.0.0.1:39441
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,214 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49046baa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:53,215 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3b18e233{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:53,215 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:53,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20237619{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:53,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d796cca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:53,221 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:53,221 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:53,221 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:53,221 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122666517-172.17.0.2-1733450018813 (Datanode Uuid b8e9c4ac-20dc-422c-8aa0-55604f5d7e84) service to localhost/127.0.0.1:39825
2024-12-06T01:53:53,222 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data3/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:53,222 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data4/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:53,222 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:53,234 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:53,237 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:53,243 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:53,243 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:53,243 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:53:53,244 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62961604{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:53,244 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55455f49{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:53,366 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d44a749{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-46073-hadoop-hdfs-3_4_1-tests_jar-_-any-14932981839308935697/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:53,367 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ce5913{HTTP/1.1, (http/1.1)}{localhost:46073}
2024-12-06T01:53:53,367 INFO  [Time-limited test {}] server.Server(415): Started @166635ms
2024-12-06T01:53:53,368 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:53,394 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:53,394 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:53,394 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:53,394 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:60596 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39943:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60596 dst: /127.0.0.1:39943
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,394 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:60582 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39943:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60582 dst: /127.0.0.1:39943
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,394 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2015138003_22 at /127.0.0.1:60604 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39943:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60604 dst: /127.0.0.1:39943
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:53,399 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c33f78d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:53,400 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68d35ecb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:53,400 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:53,400 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53068b03{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:53,400 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18af7984{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:53,402 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:53,402 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:53,402 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122666517-172.17.0.2-1733450018813 (Datanode Uuid 82b751bc-6602-436d-86e2-4249b42b23ac) service to localhost/127.0.0.1:39825
2024-12-06T01:53:53,402 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:53,404 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data1/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:53,404 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data2/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:53,404 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:53,420 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:53,424 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:53,426 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:53,427 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:53,427 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:53:53,428 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f43f081{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:53,428 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fbff314{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:53,486 WARN  [Thread-1342 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:53,489 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1bca3ce88bcdf53 with lease ID 0x2a11476020524bac: from storage DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c node DatanodeRegistration(127.0.0.1:44485, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=34689, infoSecurePort=0, ipcPort=37565, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:53:53,489 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1bca3ce88bcdf53 with lease ID 0x2a11476020524bac: from storage DS-62337f26-659f-494e-9516-552e9c59eb05 node DatanodeRegistration(127.0.0.1:44485, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=34689, infoSecurePort=0, ipcPort=37565, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:53,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20fbdc31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-42191-hadoop-hdfs-3_4_1-tests_jar-_-any-5572497579223568928/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:53,558 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1eb8f1df{HTTP/1.1, (http/1.1)}{localhost:42191}
2024-12-06T01:53:53,558 INFO  [Time-limited test {}] server.Server(415): Started @166826ms
2024-12-06T01:53:53,560 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:53,668 WARN  [Thread-1373 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:53,670 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfede0d2f9eb4fd6f with lease ID 0x2a11476020524bad: from storage DS-802f1003-ee70-40b6-8e4f-0698a5ca818b node DatanodeRegistration(127.0.0.1:33767, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=35761, infoSecurePort=0, ipcPort=45825, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:53,671 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfede0d2f9eb4fd6f with lease ID 0x2a11476020524bad: from storage DS-f27dfa02-04a0-47d7-b7e9-1f9d2985ff17 node DatanodeRegistration(127.0.0.1:33767, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=35761, infoSecurePort=0, ipcPort=45825, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:54,168 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:54,208 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:54,587 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-06T01:53:54,590 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-06T01:53:54,591 ERROR [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:54,591 WARN  [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:54,591 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C41247%2C1733450019663:(num 1733450020577) roll requested
2024-12-06T01:53:54,592 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:53:54,598 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 newFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:53:54,598 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:54,598 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:54,598 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:54,598 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:54,598 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:53:54,599 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:53:54,599 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:54,600 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:54,600 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:53:54,600 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34689:34689),(127.0.0.1/127.0.0.1:35761:35761)]
2024-12-06T01:53:54,600 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 is not closed yet, will try archiving it next time
2024-12-06T01:53:54,600 WARN  [IPC Server handler 0 on default port 39825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1015
2024-12-06T01:53:54,601 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 after 0ms
2024-12-06T01:53:55,169 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:55,208 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:56,170 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:56,209 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:56,604 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-06T01:53:57,170 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:57,210 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:57,489 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T01:53:58,171 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:58,210 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:58,601 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 after 4001ms
2024-12-06T01:53:58,608 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:33767,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:58,608 WARN  [DataStreamer for file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 block BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44485,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK], DatanodeInfoWithStorage[127.0.0.1:33767,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33767,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]) is bad.
2024-12-06T01:53:58,608 WARN  [PacketResponder: BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33767] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:58,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:33456 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:44485:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33456 dst: /127.0.0.1:44485
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:58,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:42632 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33767:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42632 dst: /127.0.0.1:33767
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:58,610 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20fbdc31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:58,610 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1eb8f1df{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:58,611 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:58,611 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fbff314{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:58,611 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f43f081{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:58,613 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:58,613 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122666517-172.17.0.2-1733450018813 (Datanode Uuid 82b751bc-6602-436d-86e2-4249b42b23ac) service to localhost/127.0.0.1:39825
2024-12-06T01:53:58,614 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data1/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:58,614 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:58,616 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data2/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:58,616 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:58,616 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:58,632 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:58,637 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:58,642 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:58,642 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:58,642 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:53:58,643 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5514f273{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:58,643 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5416c192{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:58,803 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@df3835a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-36877-hadoop-hdfs-3_4_1-tests_jar-_-any-4971693205704431260/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:58,803 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4734cdf0{HTTP/1.1, (http/1.1)}{localhost:36877}
2024-12-06T01:53:58,804 INFO  [Time-limited test {}] server.Server(415): Started @172071ms
2024-12-06T01:53:58,805 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:58,844 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:53:58,845 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-504366709_22 at /127.0.0.1:33470 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:44485:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33470 dst: /127.0.0.1:44485
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:53:58,856 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d44a749{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:58,862 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ce5913{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:53:58,862 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:53:58,862 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55455f49{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:53:58,862 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62961604{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:53:58,864 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:53:58,864 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:53:58,865 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:53:58,865 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122666517-172.17.0.2-1733450018813 (Datanode Uuid b8e9c4ac-20dc-422c-8aa0-55604f5d7e84) service to localhost/127.0.0.1:39825
2024-12-06T01:53:58,865 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data3/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:58,866 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data4/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:53:58,866 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:53:58,894 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:53:58,898 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:53:58,904 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:53:58,905 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:53:58,905 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:53:58,905 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4830599{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:53:58,906 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@295e1e6a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:53:58,961 WARN  [Thread-1416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:58,964 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3696c73637c3d1c3 with lease ID 0x2a11476020524bae: from storage DS-802f1003-ee70-40b6-8e4f-0698a5ca818b node DatanodeRegistration(127.0.0.1:38065, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=33085, infoSecurePort=0, ipcPort=36407, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:58,964 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3696c73637c3d1c3 with lease ID 0x2a11476020524bae: from storage DS-f27dfa02-04a0-47d7-b7e9-1f9d2985ff17 node DatanodeRegistration(127.0.0.1:38065, datanodeUuid=82b751bc-6602-436d-86e2-4249b42b23ac, infoPort=33085, infoSecurePort=0, ipcPort=36407, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:59,045 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@673c2578{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/java.io.tmpdir/jetty-localhost-44477-hadoop-hdfs-3_4_1-tests_jar-_-any-3201505197189010760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:53:59,045 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34440753{HTTP/1.1, (http/1.1)}{localhost:44477}
2024-12-06T01:53:59,045 INFO  [Time-limited test {}] server.Server(415): Started @172313ms
2024-12-06T01:53:59,047 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:53:59,167 WARN  [Thread-1447 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:53:59,172 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:53:59,177 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c66e1ad426f831d with lease ID 0x2a11476020524baf: from storage DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c node DatanodeRegistration(127.0.0.1:43919, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=42721, infoSecurePort=0, ipcPort=35419, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:59,177 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c66e1ad426f831d with lease ID 0x2a11476020524baf: from storage DS-62337f26-659f-494e-9516-552e9c59eb05 node DatanodeRegistration(127.0.0.1:43919, datanodeUuid=b8e9c4ac-20dc-422c-8aa0-55604f5d7e84, infoPort=42721, infoSecurePort=0, ipcPort=35419, storageInfo=lv=-57;cid=testClusterID;nsid=1702937128;c=1733450018813), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:53:59,211 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:00,086 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-06T01:54:00,089 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-06T01:54:00,090 ERROR [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44485,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:00,090 WARN  [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44485,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:00,090 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C41247%2C1733450019663:(num 1733450034591) roll requested
2024-12-06T01:54:00,091 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.1733450040091
2024-12-06T01:54:00,103 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 newFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091
2024-12-06T01:54:00,103 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:00,103 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:00,104 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:00,104 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:00,104 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:00,104 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091
2024-12-06T01:54:00,104 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44485,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:00,105 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44485,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:00,105 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:54:00,105 WARN  [IPC Server handler 2 on default port 39825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-06T01:54:00,105 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 after 0ms
2024-12-06T01:54:00,115 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42721:42721),(127.0.0.1/127.0.0.1:33085:33085)]
2024-12-06T01:54:00,115 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 is not closed yet, will try archiving it next time
2024-12-06T01:54:00,172 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:00,212 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:01,173 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:01,212 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:02,117 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:02,123 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091 newFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:02,123 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:02,123 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:02,123 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:02,124 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:02,124 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:02,124 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:02,125 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42721:42721),(127.0.0.1/127.0.0.1:33085:33085)]
2024-12-06T01:54:02,125 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 is not closed yet, will try archiving it next time
2024-12-06T01:54:02,125 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091 is not closed yet, will try archiving it next time
2024-12-06T01:54:02,125 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:54:02,126 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:54:02,126 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741838_1019 (size=1264)
2024-12-06T01:54:02,126 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577 after 0ms
2024-12-06T01:54:02,127 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:54:02,127 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 is not closed yet, will try archiving it next time
2024-12-06T01:54:02,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741838_1019 (size=1264)
2024-12-06T01:54:02,140 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733450021483/Put/vlen=218/seqid=0]
2024-12-06T01:54:02,141 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733450031207/Put/vlen=1045/seqid=0]
2024-12-06T01:54:02,141 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450020577
2024-12-06T01:54:02,141 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:54:02,141 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:54:02,141 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 after 0ms
2024-12-06T01:54:02,141 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:54:02,145 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733450034591/Put/vlen=1045/seqid=0]
2024-12-06T01:54:02,145 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733450036605/Put/vlen=1045/seqid=0]
2024-12-06T01:54:02,145 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591
2024-12-06T01:54:02,146 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091
2024-12-06T01:54:02,146 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091
2024-12-06T01:54:02,146 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091 after 0ms
2024-12-06T01:54:02,146 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450040091
2024-12-06T01:54:02,149 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733450040090/Put/vlen=1045/seqid=0]
2024-12-06T01:54:02,149 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:02,149 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:02,150 WARN  [IPC Server handler 3 on default port 39825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-06T01:54:02,150 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 after 0ms
2024-12-06T01:54:02,174 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:02,213 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:02,968 WARN  [ResponseProcessor for block BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:02,968 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2015138003_22 at /127.0.0.1:49608 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:43919:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49608 dst: /127.0.0.1:43919
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43919 remote=/127.0.0.1:49608]. Total timeout mills is 60000, 59154 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:54:02,969 WARN  [DataStreamer for file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 block BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43919,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK], DatanodeInfoWithStorage[127.0.0.1:38065,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43919,DS-abd60f34-9b7c-4728-bc9a-e273c1ad8a9c,DISK]) is bad.
2024-12-06T01:54:02,969 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2015138003_22 at /127.0.0.1:45260 [Receiving block BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:38065:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45260 dst: /127.0.0.1:38065
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:54:02,970 WARN  [DataStreamer for file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 block BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:02,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741839_1022 (size=85)
2024-12-06T01:54:03,175 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:03,214 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:03,965 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T01:54:04,106 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450034591 after 4001ms
2024-12-06T01:54:04,175 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:04,215 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:05,176 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:05,215 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:06,151 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 after 4001ms
2024-12-06T01:54:06,151 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:06,156 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:06,157 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-06T01:54:06,157 ERROR [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,157 WARN  [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,158 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C41247%2C1733450019663.meta:.meta(num 1733450020929) roll requested
2024-12-06T01:54:06,158 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.meta.1733450046158.meta
2024-12-06T01:54:06,168 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,168 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,168 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,168 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,169 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,169 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450046158.meta
2024-12-06T01:54:06,170 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,170 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,170 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta
2024-12-06T01:54:06,171 WARN  [IPC Server handler 0 on default port 39825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014
2024-12-06T01:54:06,171 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta after 1ms
2024-12-06T01:54:06,173 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33085:33085),(127.0.0.1/127.0.0.1:42721:42721)]
2024-12-06T01:54:06,173 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta is not closed yet, will try archiving it next time
2024-12-06T01:54:06,177 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:06,197 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/info/af10e94737fb415b94a93cfd14e45fb3 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80./info:regioninfo/1733450021488/Put/seqid=0
2024-12-06T01:54:06,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741841_1025 (size=7125)
2024-12-06T01:54:06,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741841_1025 (size=7125)
2024-12-06T01:54:06,204 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/info/af10e94737fb415b94a93cfd14e45fb3
2024-12-06T01:54:06,216 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:06,223 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/ns/28a0e64b2ac745bcb9360590f3939a62 is 43, key is default/ns:d/1733450020978/Put/seqid=0
2024-12-06T01:54:06,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741842_1026 (size=5153)
2024-12-06T01:54:06,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741842_1026 (size=5153)
2024-12-06T01:54:06,229 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/ns/28a0e64b2ac745bcb9360590f3939a62
2024-12-06T01:54:06,250 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/table/ce63922d5be1458d88e87a5dc3b1c82b is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733450021501/Put/seqid=0
2024-12-06T01:54:06,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741843_1027 (size=5438)
2024-12-06T01:54:06,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741843_1027 (size=5438)
2024-12-06T01:54:06,256 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/table/ce63922d5be1458d88e87a5dc3b1c82b
2024-12-06T01:54:06,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/info/af10e94737fb415b94a93cfd14e45fb3 as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/info/af10e94737fb415b94a93cfd14e45fb3
2024-12-06T01:54:06,268 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/info/af10e94737fb415b94a93cfd14e45fb3, entries=10, sequenceid=11, filesize=7.0 K
2024-12-06T01:54:06,269 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/ns/28a0e64b2ac745bcb9360590f3939a62 as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/ns/28a0e64b2ac745bcb9360590f3939a62
2024-12-06T01:54:06,274 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/ns/28a0e64b2ac745bcb9360590f3939a62, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T01:54:06,275 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/.tmp/table/ce63922d5be1458d88e87a5dc3b1c82b as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/table/ce63922d5be1458d88e87a5dc3b1c82b
2024-12-06T01:54:06,280 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/table/ce63922d5be1458d88e87a5dc3b1c82b, entries=2, sequenceid=11, filesize=5.3 K
2024-12-06T01:54:06,281 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=11, compaction requested=false
2024-12-06T01:54:06,281 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T01:54:06,281 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 0b85de0754ff399666bd5af46ffbcf80 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-06T01:54:06,281 ERROR [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,282 WARN  [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7-prefix:00c764f7c3aa,41247,1733450019663 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,282 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C41247%2C1733450019663:(num 1733450042117) roll requested
2024-12-06T01:54:06,282 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C41247%2C1733450019663.1733450046282
2024-12-06T01:54:06,286 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 newFile=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450046282
2024-12-06T01:54:06,287 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,287 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,287 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,287 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,287 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,287 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450046282
2024-12-06T01:54:06,287 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,287 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2122666517-172.17.0.2-1733450018813:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:06,288 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:06,288 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 after 0ms
2024-12-06T01:54:06,292 DEBUG [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42721:42721),(127.0.0.1/127.0.0.1:33085:33085)]
2024-12-06T01:54:06,292 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.1733450042117 to hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs/00c764f7c3aa%2C41247%2C1733450019663.1733450042117
2024-12-06T01:54:06,308 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/.tmp/info/bc84983705b3480aa40e5ad0d877b19f is 1080, key is row1002/info:/1733450031207/Put/seqid=0
2024-12-06T01:54:06,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741845_1029 (size=9270)
2024-12-06T01:54:06,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741845_1029 (size=9270)
2024-12-06T01:54:06,716 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/.tmp/info/bc84983705b3480aa40e5ad0d877b19f
2024-12-06T01:54:06,724 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/.tmp/info/bc84983705b3480aa40e5ad0d877b19f as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/info/bc84983705b3480aa40e5ad0d877b19f
2024-12-06T01:54:06,729 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/info/bc84983705b3480aa40e5ad0d877b19f, entries=4, sequenceid=8, filesize=9.1 K
2024-12-06T01:54:06,731 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 0b85de0754ff399666bd5af46ffbcf80 in 450ms, sequenceid=8, compaction requested=false
2024-12-06T01:54:06,731 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0b85de0754ff399666bd5af46ffbcf80:

2024-12-06T01:54:06,736 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:54:06,737 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:54:06,737 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:54:06,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:54:06,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:54:06,737 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:54:06,737 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:54:06,737 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=625419975, stopped=false
2024-12-06T01:54:06,737 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,33927,1733450019610
2024-12-06T01:54:06,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:54:06,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:54:06,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:06,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:06,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:54:06,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:54:06,741 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:54:06,741 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:54:06,741 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:54:06,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:54:06,742 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,41247,1733450019663' *****
2024-12-06T01:54:06,742 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:54:06,742 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:54:06,742 INFO  [RS:0;00c764f7c3aa:41247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:54:06,742 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:54:06,742 INFO  [RS:0;00c764f7c3aa:41247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:54:06,742 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b85de0754ff399666bd5af46ffbcf80
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,41247,1733450019663
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:41247.
2024-12-06T01:54:06,743 DEBUG [RS:0;00c764f7c3aa:41247 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:54:06,743 DEBUG [RS:0;00c764f7c3aa:41247 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b85de0754ff399666bd5af46ffbcf80, disabling compactions & flushes
2024-12-06T01:54:06,743 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80. after waiting 0 ms
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:54:06,743 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T01:54:06,743 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0b85de0754ff399666bd5af46ffbcf80=TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.}
2024-12-06T01:54:06,743 DEBUG [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1351): Waiting on 0b85de0754ff399666bd5af46ffbcf80, 1588230740
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:54:06,743 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:54:06,743 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:54:06,749 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/default/TestLogRolling-testLogRollOnPipelineRestart/0b85de0754ff399666bd5af46ffbcf80/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-06T01:54:06,751 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:54:06,751 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b85de0754ff399666bd5af46ffbcf80:
Waiting for close lock at 1733450046743Running coprocessor pre-close hooks at 1733450046743Disabling compacts and flushes for region at 1733450046743Disabling writes for close at 1733450046743Writing region close event to WAL at 1733450046744 (+1 ms)Running coprocessor post-close hooks at 1733450046750 (+6 ms)Closed at 1733450046750
2024-12-06T01:54:06,751 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733450021112.0b85de0754ff399666bd5af46ffbcf80.
2024-12-06T01:54:06,753 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T01:54:06,754 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:54:06,754 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:54:06,754 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450046743Running coprocessor pre-close hooks at 1733450046743Disabling compacts and flushes for region at 1733450046743Disabling writes for close at 1733450046743Writing region close event to WAL at 1733450046749 (+6 ms)Running coprocessor post-close hooks at 1733450046754 (+5 ms)Closed at 1733450046754
2024-12-06T01:54:06,754 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:54:06,943 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,41247,1733450019663; all regions closed.
2024-12-06T01:54:06,944 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,944 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,944 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,944 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,945 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:06,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741840_1023 (size=825)
2024-12-06T01:54:06,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741840_1023 (size=825)
2024-12-06T01:54:07,178 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:07,216 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:07,465 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-06T01:54:07,465 INFO  [regionserver/00c764f7c3aa:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-06T01:54:08,178 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:08,217 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:08,434 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:54:09,177 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T01:54:09,179 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:09,217 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:09,592 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:54:10,172 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta after 4002ms
2024-12-06T01:54:10,173 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/WALs/00c764f7c3aa,41247,1733450019663/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta to hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs/00c764f7c3aa%2C41247%2C1733450019663.meta.1733450020929.meta
2024-12-06T01:54:10,175 DEBUG [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs
2024-12-06T01:54:10,176 INFO  [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C41247%2C1733450019663.meta:.meta(num 1733450046158)
2024-12-06T01:54:10,176 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,176 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,176 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,176 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,177 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741844_1028 (size=1162)
2024-12-06T01:54:10,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741844_1028 (size=1162)
2024-12-06T01:54:10,180 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:10,183 DEBUG [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs
2024-12-06T01:54:10,183 INFO  [RS:0;00c764f7c3aa:41247 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C41247%2C1733450019663:(num 1733450046282)
2024-12-06T01:54:10,183 DEBUG [RS:0;00c764f7c3aa:41247 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:54:10,183 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:54:10,183 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:54:10,184 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T01:54:10,184 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:54:10,184 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:54:10,184 INFO  [RS:0;00c764f7c3aa:41247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41247
2024-12-06T01:54:10,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,41247,1733450019663
2024-12-06T01:54:10,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:54:10,188 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:54:10,189 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,41247,1733450019663]
2024-12-06T01:54:10,191 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,41247,1733450019663 already deleted, retry=false
2024-12-06T01:54:10,191 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,41247,1733450019663 expired; onlineServers=0
2024-12-06T01:54:10,191 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,33927,1733450019610' *****
2024-12-06T01:54:10,191 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:54:10,191 INFO  [M:0;00c764f7c3aa:33927 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:54:10,191 INFO  [M:0;00c764f7c3aa:33927 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:54:10,191 DEBUG [M:0;00c764f7c3aa:33927 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:54:10,191 DEBUG [M:0;00c764f7c3aa:33927 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:54:10,191 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:54:10,191 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450020309 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450020309,5,FailOnTimeoutGroup]
2024-12-06T01:54:10,191 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450020312 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450020312,5,FailOnTimeoutGroup]
2024-12-06T01:54:10,191 INFO  [M:0;00c764f7c3aa:33927 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:54:10,191 INFO  [M:0;00c764f7c3aa:33927 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:54:10,192 DEBUG [M:0;00c764f7c3aa:33927 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:54:10,192 INFO  [M:0;00c764f7c3aa:33927 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:54:10,192 INFO  [M:0;00c764f7c3aa:33927 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:54:10,192 INFO  [M:0;00c764f7c3aa:33927 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:54:10,192 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:54:10,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:54:10,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:10,193 DEBUG [M:0;00c764f7c3aa:33927 {}] zookeeper.ZKUtil(347): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:54:10,193 WARN  [M:0;00c764f7c3aa:33927 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:54:10,194 INFO  [M:0;00c764f7c3aa:33927 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/.lastflushedseqids
2024-12-06T01:54:10,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741846_1030 (size=120)
2024-12-06T01:54:10,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741846_1030 (size=120)
2024-12-06T01:54:10,200 INFO  [M:0;00c764f7c3aa:33927 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:54:10,200 INFO  [M:0;00c764f7c3aa:33927 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:54:10,200 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:54:10,200 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:10,201 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:10,201 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:54:10,201 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:10,201 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-12-06T01:54:10,201 ERROR [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData-prefix:00c764f7c3aa,33927,1733450019610 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:10,201 WARN  [FSHLog-0-hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData-prefix:00c764f7c3aa,33927,1733450019610 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:10,201 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 00c764f7c3aa%2C33927%2C1733450019610:(num 1733450020188) roll requested
2024-12-06T01:54:10,201 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C33927%2C1733450019610.1733450050201
2024-12-06T01:54:10,209 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,209 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,209 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,209 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,209 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,209 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450050201
2024-12-06T01:54:10,209 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:10,210 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39943,DS-802f1003-ee70-40b6-8e4f-0698a5ca818b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-06T01:54:10,210 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188
2024-12-06T01:54:10,210 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33085:33085),(127.0.0.1/127.0.0.1:42721:42721)]
2024-12-06T01:54:10,210 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 is not closed yet, will try archiving it next time
2024-12-06T01:54:10,210 WARN  [IPC Server handler 4 on default port 39825 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-12-06T01:54:10,210 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 after 0ms
2024-12-06T01:54:10,218 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:10,227 DEBUG [M:0;00c764f7c3aa:33927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2c4a00abbb94d61a7460408336e8fe9 is 82, key is hbase:meta,,1/info:regioninfo/1733450020956/Put/seqid=0
2024-12-06T01:54:10,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741848_1033 (size=5672)
2024-12-06T01:54:10,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741848_1033 (size=5672)
2024-12-06T01:54:10,232 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2c4a00abbb94d61a7460408336e8fe9
2024-12-06T01:54:10,251 DEBUG [M:0;00c764f7c3aa:33927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c1e0e18273ac452cae2b8ea2580d1d02 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733450021507/Put/seqid=0
2024-12-06T01:54:10,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741849_1034 (size=6118)
2024-12-06T01:54:10,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741849_1034 (size=6118)
2024-12-06T01:54:10,257 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c1e0e18273ac452cae2b8ea2580d1d02
2024-12-06T01:54:10,277 DEBUG [M:0;00c764f7c3aa:33927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b0c445c1c93c484ab8cf6389bfac93d3 is 69, key is 00c764f7c3aa,41247,1733450019663/rs:state/1733450020416/Put/seqid=0
2024-12-06T01:54:10,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741850_1035 (size=5156)
2024-12-06T01:54:10,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741850_1035 (size=5156)
2024-12-06T01:54:10,283 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b0c445c1c93c484ab8cf6389bfac93d3
2024-12-06T01:54:10,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:54:10,289 INFO  [RS:0;00c764f7c3aa:41247 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:54:10,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x1006507ebf10001, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:54:10,289 INFO  [RS:0;00c764f7c3aa:41247 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,41247,1733450019663; zookeeper connection closed.
2024-12-06T01:54:10,290 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@41168c86 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@41168c86
2024-12-06T01:54:10,290 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T01:54:10,303 DEBUG [M:0;00c764f7c3aa:33927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b8ec16a23b664693abf3e4ff0f43e96e is 52, key is load_balancer_on/state:d/1733450021107/Put/seqid=0
2024-12-06T01:54:10,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741851_1036 (size=5056)
2024-12-06T01:54:10,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741851_1036 (size=5056)
2024-12-06T01:54:10,308 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b8ec16a23b664693abf3e4ff0f43e96e
2024-12-06T01:54:10,314 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e2c4a00abbb94d61a7460408336e8fe9 as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e2c4a00abbb94d61a7460408336e8fe9
2024-12-06T01:54:10,319 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e2c4a00abbb94d61a7460408336e8fe9, entries=8, sequenceid=56, filesize=5.5 K
2024-12-06T01:54:10,320 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c1e0e18273ac452cae2b8ea2580d1d02 as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c1e0e18273ac452cae2b8ea2580d1d02
2024-12-06T01:54:10,324 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c1e0e18273ac452cae2b8ea2580d1d02, entries=6, sequenceid=56, filesize=6.0 K
2024-12-06T01:54:10,325 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b0c445c1c93c484ab8cf6389bfac93d3 as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b0c445c1c93c484ab8cf6389bfac93d3
2024-12-06T01:54:10,330 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b0c445c1c93c484ab8cf6389bfac93d3, entries=1, sequenceid=56, filesize=5.0 K
2024-12-06T01:54:10,331 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b8ec16a23b664693abf3e4ff0f43e96e as hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b8ec16a23b664693abf3e4ff0f43e96e
2024-12-06T01:54:10,336 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b8ec16a23b664693abf3e4ff0f43e96e, entries=1, sequenceid=56, filesize=4.9 K
2024-12-06T01:54:10,338 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=56, compaction requested=false
2024-12-06T01:54:10,339 INFO  [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:10,339 DEBUG [M:0;00c764f7c3aa:33927 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450050200Disabling compacts and flushes for region at 1733450050200Disabling writes for close at 1733450050201 (+1 ms)Obtaining lock to block concurrent updates at 1733450050201Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733450050201Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733450050201Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733450050211 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733450050211Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733450050226 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733450050226Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733450050237 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733450050250 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733450050250Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733450050263 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733450050277 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733450050277Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733450050289 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733450050302 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733450050302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60fc2d5: reopening flushed file at 1733450050313 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc890c5: reopening flushed file at 1733450050319 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ef0a362: reopening flushed file at 1733450050324 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a64566: reopening flushed file at 1733450050331 (+7 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=56, compaction requested=false at 1733450050338 (+7 ms)Writing region close event to WAL at 1733450050339 (+1 ms)Closed at 1733450050339
2024-12-06T01:54:10,339 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,340 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,340 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,340 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,340 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:10,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43919 is added to blk_1073741847_1031 (size=757)
2024-12-06T01:54:10,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38065 is added to blk_1073741847_1031 (size=757)
2024-12-06T01:54:11,180 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:11,218 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:11,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,778 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,778 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,778 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,780 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,783 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:11,783 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,178 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-06T01:54:12,181 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:12,219 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:12,286 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:54:12,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,303 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,305 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,308 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,308 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,308 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:12,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:13,181 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:13,220 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:13,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:54:13,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:54:13,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T01:54:13,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-06T01:54:14,182 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:14,211 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 after 4001ms
2024-12-06T01:54:14,212 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/WALs/00c764f7c3aa,33927,1733450019610/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 to hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/oldWALs/00c764f7c3aa%2C33927%2C1733450019610.1733450020188
2024-12-06T01:54:14,215 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/MasterData/oldWALs/00c764f7c3aa%2C33927%2C1733450019610.1733450020188 to hdfs://localhost:39825/user/jenkins/test-data/9270c063-49d6-bb64-d39d-9ebe9bb405c7/oldWALs/00c764f7c3aa%2C33927%2C1733450019610.1733450020188$masterlocalwal$
2024-12-06T01:54:14,215 INFO  [M:0;00c764f7c3aa:33927 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:54:14,215 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:54:14,215 INFO  [M:0;00c764f7c3aa:33927 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33927
2024-12-06T01:54:14,215 INFO  [M:0;00c764f7c3aa:33927 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:54:14,220 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:14,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:54:14,317 INFO  [M:0;00c764f7c3aa:33927 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:54:14,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33927-0x1006507ebf10000, quorum=127.0.0.1:54846, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:54:14,320 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@673c2578{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:54:14,320 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34440753{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:54:14,320 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:54:14,320 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@295e1e6a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:54:14,320 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4830599{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:54:14,322 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:54:14,322 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122666517-172.17.0.2-1733450018813 (Datanode Uuid b8e9c4ac-20dc-422c-8aa0-55604f5d7e84) service to localhost/127.0.0.1:39825
2024-12-06T01:54:14,322 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:54:14,322 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:54:14,322 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data3/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:54:14,322 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data4/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:54:14,323 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:54:14,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@df3835a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:54:14,325 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4734cdf0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:54:14,325 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:54:14,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5416c192{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:54:14,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5514f273{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:54:14,326 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:54:14,326 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:54:14,326 WARN  [BP-2122666517-172.17.0.2-1733450018813 heartbeating to localhost/127.0.0.1:39825 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2122666517-172.17.0.2-1733450018813 (Datanode Uuid 82b751bc-6602-436d-86e2-4249b42b23ac) service to localhost/127.0.0.1:39825
2024-12-06T01:54:14,326 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:54:14,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data1/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:54:14,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/cluster_37594739-a9df-18df-ce39-84e071796833/data/data2/current/BP-2122666517-172.17.0.2-1733450018813 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:54:14,327 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:54:14,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c4c555c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:54:14,333 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30337913{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:54:14,333 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:54:14,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2eab1cda{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:54:14,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e676e1c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir/,STOPPED}
2024-12-06T01:54:14,340 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:54:14,357 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:54:14,367 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154)
Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:39825 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:39825
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:39825 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:39825 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:39825
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39825
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=455 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 355), ProcessCount=11 (was 11), AvailableMemoryMB=9711 (was 9881)
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=9712
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.log.dir so I do NOT create it in target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/95ae87a6-dc78-2bf6-7268-3bf6dddac1de/hadoop.tmp.dir so I do NOT create it in target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a, deleteOnExit=true
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/test.cache.data in system properties and HBase conf
2024-12-06T01:54:14,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:54:14,377 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:54:14,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:54:14,378 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:54:14,378 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:54:14,378 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:54:14,378 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:54:14,390 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:54:14,459 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:54:14,464 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:54:14,465 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:54:14,465 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:54:14,465 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:54:14,466 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:54:14,467 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67166726{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:54:14,467 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1df61fd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:54:14,590 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ee3adbc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/java.io.tmpdir/jetty-localhost-33861-hadoop-hdfs-3_4_1-tests_jar-_-any-12326050745865498418/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:54:14,591 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1210b54d{HTTP/1.1, (http/1.1)}{localhost:33861}
2024-12-06T01:54:14,591 INFO  [Time-limited test {}] server.Server(415): Started @187859ms
2024-12-06T01:54:14,604 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:54:14,654 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:54:14,656 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:54:14,657 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:54:14,657 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:54:14,657 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:54:14,658 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@418725d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:54:14,658 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24056a85{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:54:14,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@63c886b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/java.io.tmpdir/jetty-localhost-39215-hadoop-hdfs-3_4_1-tests_jar-_-any-12438490119089924801/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:54:14,774 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e71ae2f{HTTP/1.1, (http/1.1)}{localhost:39215}
2024-12-06T01:54:14,774 INFO  [Time-limited test {}] server.Server(415): Started @188042ms
2024-12-06T01:54:14,776 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:54:14,806 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:54:14,810 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:54:14,810 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:54:14,810 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:54:14,810 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:54:14,811 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@667e76c4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:54:14,811 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@392e4f56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:54:14,852 WARN  [Thread-1641 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data1/current/BP-2074420978-172.17.0.2-1733450054407/current, will proceed with Du for space computation calculation, 
2024-12-06T01:54:14,852 WARN  [Thread-1642 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data2/current/BP-2074420978-172.17.0.2-1733450054407/current, will proceed with Du for space computation calculation, 
2024-12-06T01:54:14,868 WARN  [Thread-1620 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:54:14,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47ec76044ae84116 with lease ID 0xdcfac62dbf190a5e: Processing first storage report for DS-ced9970f-5115-4461-bc2b-a685b09a6228 from datanode DatanodeRegistration(127.0.0.1:39353, datanodeUuid=1cb5fa48-0475-4200-b9b1-f8a963cd20bf, infoPort=45839, infoSecurePort=0, ipcPort=32895, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407)
2024-12-06T01:54:14,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47ec76044ae84116 with lease ID 0xdcfac62dbf190a5e: from storage DS-ced9970f-5115-4461-bc2b-a685b09a6228 node DatanodeRegistration(127.0.0.1:39353, datanodeUuid=1cb5fa48-0475-4200-b9b1-f8a963cd20bf, infoPort=45839, infoSecurePort=0, ipcPort=32895, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:54:14,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47ec76044ae84116 with lease ID 0xdcfac62dbf190a5e: Processing first storage report for DS-a67ace9e-6a4f-46e3-ad22-b921c9c64479 from datanode DatanodeRegistration(127.0.0.1:39353, datanodeUuid=1cb5fa48-0475-4200-b9b1-f8a963cd20bf, infoPort=45839, infoSecurePort=0, ipcPort=32895, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407)
2024-12-06T01:54:14,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47ec76044ae84116 with lease ID 0xdcfac62dbf190a5e: from storage DS-a67ace9e-6a4f-46e3-ad22-b921c9c64479 node DatanodeRegistration(127.0.0.1:39353, datanodeUuid=1cb5fa48-0475-4200-b9b1-f8a963cd20bf, infoPort=45839, infoSecurePort=0, ipcPort=32895, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:54:14,953 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1780e3e3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/java.io.tmpdir/jetty-localhost-36949-hadoop-hdfs-3_4_1-tests_jar-_-any-5960320390463071387/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:54:14,953 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c93132a{HTTP/1.1, (http/1.1)}{localhost:36949}
2024-12-06T01:54:14,953 INFO  [Time-limited test {}] server.Server(415): Started @188221ms
2024-12-06T01:54:14,955 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:54:15,030 WARN  [Thread-1667 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data3/current/BP-2074420978-172.17.0.2-1733450054407/current, will proceed with Du for space computation calculation, 
2024-12-06T01:54:15,030 WARN  [Thread-1668 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data4/current/BP-2074420978-172.17.0.2-1733450054407/current, will proceed with Du for space computation calculation, 
2024-12-06T01:54:15,049 WARN  [Thread-1656 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:54:15,051 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7bc13170e520c444 with lease ID 0xdcfac62dbf190a5f: Processing first storage report for DS-7d1cd802-0a9f-421d-a514-5f51e8cfc4c9 from datanode DatanodeRegistration(127.0.0.1:45057, datanodeUuid=38721538-9187-4051-a36d-31cce0b7b9e9, infoPort=46711, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407)
2024-12-06T01:54:15,051 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7bc13170e520c444 with lease ID 0xdcfac62dbf190a5f: from storage DS-7d1cd802-0a9f-421d-a514-5f51e8cfc4c9 node DatanodeRegistration(127.0.0.1:45057, datanodeUuid=38721538-9187-4051-a36d-31cce0b7b9e9, infoPort=46711, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:54:15,051 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7bc13170e520c444 with lease ID 0xdcfac62dbf190a5f: Processing first storage report for DS-18b00cc8-de79-435a-86d8-f3cd45acf8af from datanode DatanodeRegistration(127.0.0.1:45057, datanodeUuid=38721538-9187-4051-a36d-31cce0b7b9e9, infoPort=46711, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407)
2024-12-06T01:54:15,051 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7bc13170e520c444 with lease ID 0xdcfac62dbf190a5f: from storage DS-18b00cc8-de79-435a-86d8-f3cd45acf8af node DatanodeRegistration(127.0.0.1:45057, datanodeUuid=38721538-9187-4051-a36d-31cce0b7b9e9, infoPort=46711, infoSecurePort=0, ipcPort=41493, storageInfo=lv=-57;cid=testClusterID;nsid=1449908996;c=1733450054407), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:54:15,077 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8
2024-12-06T01:54:15,080 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/zookeeper_0, clientPort=61114, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:54:15,081 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61114
2024-12-06T01:54:15,081 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,083 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:54:15,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:54:15,092 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297 with version=8
2024-12-06T01:54:15,093 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:54:15,095 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:54:15,095 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:54:15,095 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:54:15,095 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:54:15,095 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:54:15,095 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:54:15,096 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:54:15,096 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:54:15,096 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34381
2024-12-06T01:54:15,098 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34381 connecting to ZooKeeper ensemble=127.0.0.1:61114
2024-12-06T01:54:15,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:343810x0, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:54:15,104 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34381-0x1006508768d0000 connected
2024-12-06T01:54:15,123 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,124 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:54:15,126 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297, hbase.cluster.distributed=false
2024-12-06T01:54:15,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:54:15,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34381
2024-12-06T01:54:15,129 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34381
2024-12-06T01:54:15,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34381
2024-12-06T01:54:15,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34381
2024-12-06T01:54:15,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34381
2024-12-06T01:54:15,147 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:54:15,147 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:54:15,147 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:54:15,148 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:54:15,148 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:54:15,148 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:54:15,148 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:54:15,148 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:54:15,148 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42833
2024-12-06T01:54:15,150 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42833 connecting to ZooKeeper ensemble=127.0.0.1:61114
2024-12-06T01:54:15,150 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,152 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428330x0, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:54:15,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:54:15,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42833-0x1006508768d0001 connected
2024-12-06T01:54:15,160 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:54:15,161 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:54:15,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:54:15,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:54:15,164 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42833
2024-12-06T01:54:15,164 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42833
2024-12-06T01:54:15,164 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42833
2024-12-06T01:54:15,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42833
2024-12-06T01:54:15,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42833
2024-12-06T01:54:15,179 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:34381
2024-12-06T01:54:15,179 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:54:15,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:54:15,182 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,183 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:15,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:54:15,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,184 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:54:15,185 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,34381,1733450055095 from backup master directory
2024-12-06T01:54:15,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:54:15,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:54:15,186 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:54:15,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,191 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/hbase.id] with ID: 887f6f0d-2698-4b42-b171-f9ea586a47ac
2024-12-06T01:54:15,191 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/.tmp/hbase.id
2024-12-06T01:54:15,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:54:15,198 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/.tmp/hbase.id]:[hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/hbase.id]
2024-12-06T01:54:15,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:54:15,214 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:15,214 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:54:15,215 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T01:54:15,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,221 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:15,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:54:15,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:54:15,226 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:54:15,227 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:54:15,227 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:54:15,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:54:15,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:54:15,238 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store
2024-12-06T01:54:15,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:54:15,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:54:15,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:54:15,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:54:15,248 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:15,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:15,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:54:15,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:15,248 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:54:15,248 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450055248Disabling compacts and flushes for region at 1733450055248Disabling writes for close at 1733450055248Writing region close event to WAL at 1733450055248Closed at 1733450055248
2024-12-06T01:54:15,249 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/.initializing
2024-12-06T01:54:15,249 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/WALs/00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,251 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C34381%2C1733450055095, suffix=, logDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/WALs/00c764f7c3aa,34381,1733450055095, archiveDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/oldWALs, maxLogs=10
2024-12-06T01:54:15,252 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C34381%2C1733450055095.1733450055252
2024-12-06T01:54:15,256 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/WALs/00c764f7c3aa,34381,1733450055095/00c764f7c3aa%2C34381%2C1733450055095.1733450055252
2024-12-06T01:54:15,260 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45839:45839),(127.0.0.1/127.0.0.1:46711:46711)]
2024-12-06T01:54:15,261 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:54:15,261 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:54:15,261 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,261 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,262 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:54:15,264 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:54:15,264 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,264 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,264 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:54:15,265 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:54:15,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,266 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:54:15,266 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:54:15,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:54:15,267 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:54:15,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:54:15,268 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:54:15,268 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,269 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:54:15,269 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,270 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,270 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,271 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,271 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,272 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:54:15,273 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:54:15,275 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:54:15,275 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=880994, jitterRate=0.12024189531803131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:54:15,276 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733450055261Initializing all the Stores at 1733450055262 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055262Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450055262Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450055262Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450055262Cleaning up temporary data from old regions at 1733450055271 (+9 ms)Region opened successfully at 1733450055276 (+5 ms)
2024-12-06T01:54:15,276 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:54:15,279 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707dc676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:54:15,280 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:54:15,280 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:54:15,280 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:54:15,280 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:54:15,281 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T01:54:15,281 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T01:54:15,281 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:54:15,283 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:54:15,284 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:54:15,286 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:54:15,286 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:54:15,287 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:54:15,288 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:54:15,289 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:54:15,290 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:54:15,291 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:54:15,292 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:54:15,293 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:54:15,294 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:54:15,296 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:54:15,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:54:15,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:54:15,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,299 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,34381,1733450055095, sessionid=0x1006508768d0000, setting cluster-up flag (Was=false)
2024-12-06T01:54:15,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,307 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:54:15,308 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,315 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:54:15,316 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:15,317 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:54:15,319 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:54:15,319 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:54:15,320 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:54:15,320 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,34381,1733450055095
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:54:15,321 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733450085322
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:54:15,322 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,323 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:54:15,323 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:54:15,323 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:54:15,323 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:54:15,323 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:54:15,324 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:54:15,324 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:54:15,324 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450055324,5,FailOnTimeoutGroup]
2024-12-06T01:54:15,324 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,324 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450055324,5,FailOnTimeoutGroup]
2024-12-06T01:54:15,324 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,324 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:54:15,324 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,324 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,324 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:54:15,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:54:15,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:54:15,331 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:54:15,332 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297
2024-12-06T01:54:15,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:54:15,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:54:15,338 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:54:15,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:54:15,340 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:54:15,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:54:15,342 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:54:15,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:54:15,343 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:54:15,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,344 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,344 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:54:15,345 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:54:15,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,345 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:54:15,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740
2024-12-06T01:54:15,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740
2024-12-06T01:54:15,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:54:15,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:54:15,348 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:54:15,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:54:15,350 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:54:15,350 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812211, jitterRate=0.03277990221977234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:54:15,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733450055338Initializing all the Stores at 1733450055339 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055339Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055339Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450055339Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055339Cleaning up temporary data from old regions at 1733450055347 (+8 ms)Region opened successfully at 1733450055351 (+4 ms)
2024-12-06T01:54:15,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:54:15,351 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:54:15,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:54:15,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:54:15,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:54:15,352 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:54:15,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450055351Disabling compacts and flushes for region at 1733450055351Disabling writes for close at 1733450055351Writing region close event to WAL at 1733450055352 (+1 ms)Closed at 1733450055352
2024-12-06T01:54:15,353 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:54:15,353 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:54:15,353 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:54:15,354 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:54:15,355 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:54:15,369 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(746): ClusterId : 887f6f0d-2698-4b42-b171-f9ea586a47ac
2024-12-06T01:54:15,369 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:54:15,372 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:54:15,372 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:54:15,375 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:54:15,375 DEBUG [RS:0;00c764f7c3aa:42833 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f1981a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:54:15,393 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:42833
2024-12-06T01:54:15,393 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:54:15,393 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:54:15,393 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:54:15,394 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,34381,1733450055095 with port=42833, startcode=1733450055147
2024-12-06T01:54:15,394 DEBUG [RS:0;00c764f7c3aa:42833 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:54:15,397 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:54:15,398 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34381 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,398 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34381 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,400 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297
2024-12-06T01:54:15,400 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38289
2024-12-06T01:54:15,400 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:54:15,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:54:15,402 DEBUG [RS:0;00c764f7c3aa:42833 {}] zookeeper.ZKUtil(111): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,402 WARN  [RS:0;00c764f7c3aa:42833 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:54:15,402 INFO  [RS:0;00c764f7c3aa:42833 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:54:15,403 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,403 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,42833,1733450055147]
2024-12-06T01:54:15,406 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:54:15,409 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:54:15,409 INFO  [RS:0;00c764f7c3aa:42833 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:54:15,409 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,410 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:54:15,411 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:54:15,411 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:54:15,411 DEBUG [RS:0;00c764f7c3aa:42833 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:54:15,413 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,413 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,413 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,413 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,413 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,413 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42833,1733450055147-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:54:15,436 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:54:15,436 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42833,1733450055147-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,436 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,436 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.Replication(171): 00c764f7c3aa,42833,1733450055147 started
2024-12-06T01:54:15,455 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,455 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,42833,1733450055147, RpcServer on 00c764f7c3aa/172.17.0.2:42833, sessionid=0x1006508768d0001
2024-12-06T01:54:15,455 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:54:15,455 DEBUG [RS:0;00c764f7c3aa:42833 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,455 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,42833,1733450055147'
2024-12-06T01:54:15,455 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:54:15,456 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:54:15,456 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:54:15,456 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:54:15,456 DEBUG [RS:0;00c764f7c3aa:42833 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,456 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,42833,1733450055147'
2024-12-06T01:54:15,456 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:54:15,457 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:54:15,457 DEBUG [RS:0;00c764f7c3aa:42833 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:54:15,457 INFO  [RS:0;00c764f7c3aa:42833 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:54:15,457 INFO  [RS:0;00c764f7c3aa:42833 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:54:15,506 WARN  [00c764f7c3aa:34381 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:54:15,559 INFO  [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C42833%2C1733450055147, suffix=, logDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147, archiveDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/oldWALs, maxLogs=32
2024-12-06T01:54:15,560 INFO  [RS:0;00c764f7c3aa:42833 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42833%2C1733450055147.1733450055560
2024-12-06T01:54:15,566 INFO  [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450055560
2024-12-06T01:54:15,570 DEBUG [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46711:46711),(127.0.0.1/127.0.0.1:45839:45839)]
2024-12-06T01:54:15,756 DEBUG [00c764f7c3aa:34381 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:54:15,757 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,758 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,42833,1733450055147, state=OPENING
2024-12-06T01:54:15,760 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:54:15,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:54:15,762 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:54:15,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:54:15,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:54:15,762 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,42833,1733450055147}]
2024-12-06T01:54:15,916 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:54:15,918 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58121, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:54:15,921 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:54:15,921 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:54:15,924 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C42833%2C1733450055147.meta, suffix=.meta, logDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147, archiveDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/oldWALs, maxLogs=32
2024-12-06T01:54:15,924 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42833%2C1733450055147.meta.1733450055924.meta
2024-12-06T01:54:15,929 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.meta.1733450055924.meta
2024-12-06T01:54:15,931 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45839:45839),(127.0.0.1/127.0.0.1:46711:46711)]
2024-12-06T01:54:15,931 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:54:15,932 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:54:15,932 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:54:15,932 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:54:15,932 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:54:15,932 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:54:15,932 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:54:15,932 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:54:15,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:54:15,935 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:54:15,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,935 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,935 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:54:15,936 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:54:15,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:54:15,937 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:54:15,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,937 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,937 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:54:15,938 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:54:15,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:15,938 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:54:15,938 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:54:15,939 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740
2024-12-06T01:54:15,940 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740
2024-12-06T01:54:15,941 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:54:15,941 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:54:15,941 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:54:15,943 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:54:15,943 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735937, jitterRate=-0.06420877575874329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:54:15,943 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:54:15,944 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733450055933Writing region info on filesystem at 1733450055933Initializing all the Stores at 1733450055933Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055933Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055933Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450055934 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450055934Cleaning up temporary data from old regions at 1733450055941 (+7 ms)Running coprocessor post-open hooks at 1733450055943 (+2 ms)Region opened successfully at 1733450055944 (+1 ms)
2024-12-06T01:54:15,945 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733450055915
2024-12-06T01:54:15,948 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:54:15,948 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:54:15,949 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,950 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,42833,1733450055147, state=OPEN
2024-12-06T01:54:15,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:54:15,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:54:15,954 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:15,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:54:15,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:54:15,957 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:54:15,957 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,42833,1733450055147 in 192 msec
2024-12-06T01:54:15,959 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:54:15,959 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 604 msec
2024-12-06T01:54:15,960 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:54:15,960 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:54:15,962 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:54:15,962 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,42833,1733450055147, seqNum=-1]
2024-12-06T01:54:15,962 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:54:15,964 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:54:15,969 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 650 msec
2024-12-06T01:54:15,969 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733450055969, completionTime=-1
2024-12-06T01:54:15,970 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:54:15,970 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:54:15,971 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:54:15,971 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733450115971
2024-12-06T01:54:15,971 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733450175971
2024-12-06T01:54:15,971 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-06T01:54:15,972 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34381,1733450055095-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,972 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34381,1733450055095-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,972 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34381,1733450055095-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,972 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:34381, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,972 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,972 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:15,973 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:54:15,975 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.789sec
2024-12-06T01:54:15,975 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:54:15,975 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:54:15,975 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:54:15,976 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:54:15,976 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:54:15,976 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34381,1733450055095-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:54:15,976 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34381,1733450055095-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:54:15,978 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:54:15,978 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:54:15,978 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34381,1733450055095-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:16,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d2fdf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:54:16,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,34381,-1 for getting cluster id
2024-12-06T01:54:16,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:54:16,072 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '887f6f0d-2698-4b42-b171-f9ea586a47ac'
2024-12-06T01:54:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:54:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "887f6f0d-2698-4b42-b171-f9ea586a47ac"

2024-12-06T01:54:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23273074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:54:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,34381,-1]
2024-12-06T01:54:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:54:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:54:16,074 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:54:16,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726fccb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:54:16,076 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:54:16,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,42833,1733450055147, seqNum=-1]
2024-12-06T01:54:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:54:16,078 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:54:16,079 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:16,080 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:54:16,082 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:54:16,083 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T01:54:16,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,34381,1733450055095
2024-12-06T01:54:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1fe576c5
2024-12-06T01:54:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T01:54:16,085 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T01:54:16,085 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T01:54:16,085 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T01:54:16,086 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:54:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:16,089 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T01:54:16,089 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:16,089 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-06T01:54:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:54:16,090 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T01:54:16,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741835_1011 (size=405)
2024-12-06T01:54:16,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741835_1011 (size=405)
2024-12-06T01:54:16,183 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:16,221 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:16,500 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4ba57513049c73ee0fe02636937e1079, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297
2024-12-06T01:54:16,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741836_1012 (size=88)
2024-12-06T01:54:16,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741836_1012 (size=88)
2024-12-06T01:54:16,508 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:54:16,508 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 4ba57513049c73ee0fe02636937e1079, disabling compactions & flushes
2024-12-06T01:54:16,509 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079. after waiting 0 ms
2024-12-06T01:54:16,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,509 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,509 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4ba57513049c73ee0fe02636937e1079:
Waiting for close lock at 1733450056508Disabling compacts and flushes for region at 1733450056508Disabling writes for close at 1733450056509 (+1 ms)Writing region close event to WAL at 1733450056509Closed at 1733450056509
2024-12-06T01:54:16,511 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T01:54:16,511 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733450056511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733450056511"}]},"ts":"1733450056511"}
2024-12-06T01:54:16,514 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T01:54:16,516 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T01:54:16,516 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733450056516"}]},"ts":"1733450056516"}
2024-12-06T01:54:16,518 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-06T01:54:16,519 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=4ba57513049c73ee0fe02636937e1079, ASSIGN}]
2024-12-06T01:54:16,520 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=4ba57513049c73ee0fe02636937e1079, ASSIGN
2024-12-06T01:54:16,522 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=4ba57513049c73ee0fe02636937e1079, ASSIGN; state=OFFLINE, location=00c764f7c3aa,42833,1733450055147; forceNewPlan=false, retain=false
2024-12-06T01:54:16,672 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4ba57513049c73ee0fe02636937e1079, regionState=OPENING, regionLocation=00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:16,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=4ba57513049c73ee0fe02636937e1079, ASSIGN because future has completed
2024-12-06T01:54:16,677 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ba57513049c73ee0fe02636937e1079, server=00c764f7c3aa,42833,1733450055147}]
2024-12-06T01:54:16,834 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,834 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ba57513049c73ee0fe02636937e1079, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:54:16,834 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,834 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:54:16,835 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,835 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,836 INFO  [StoreOpener-4ba57513049c73ee0fe02636937e1079-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4ba57513049c73ee0fe02636937e1079 
2024-12-06T01:54:16,838 INFO  [StoreOpener-4ba57513049c73ee0fe02636937e1079-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba57513049c73ee0fe02636937e1079 columnFamilyName info
2024-12-06T01:54:16,838 DEBUG [StoreOpener-4ba57513049c73ee0fe02636937e1079-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:54:16,838 INFO  [StoreOpener-4ba57513049c73ee0fe02636937e1079-1 {}] regionserver.HStore(327): Store=4ba57513049c73ee0fe02636937e1079/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:54:16,838 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,839 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,839 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,840 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,840 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,841 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,843 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:54:16,843 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4ba57513049c73ee0fe02636937e1079; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876933, jitterRate=0.11507891118526459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:54:16,843 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:54:16,844 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4ba57513049c73ee0fe02636937e1079:
Running coprocessor pre-open hook at 1733450056835Writing region info on filesystem at 1733450056835Initializing all the Stores at 1733450056836 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450056836Cleaning up temporary data from old regions at 1733450056840 (+4 ms)Running coprocessor post-open hooks at 1733450056843 (+3 ms)Region opened successfully at 1733450056844 (+1 ms)
2024-12-06T01:54:16,845 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079., pid=6, masterSystemTime=1733450056830
2024-12-06T01:54:16,847 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,848 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:16,849 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4ba57513049c73ee0fe02636937e1079, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,42833,1733450055147
2024-12-06T01:54:16,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ba57513049c73ee0fe02636937e1079, server=00c764f7c3aa,42833,1733450055147 because future has completed
2024-12-06T01:54:16,855 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T01:54:16,855 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ba57513049c73ee0fe02636937e1079, server=00c764f7c3aa,42833,1733450055147 in 176 msec
2024-12-06T01:54:16,858 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T01:54:16,858 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=4ba57513049c73ee0fe02636937e1079, ASSIGN in 336 msec
2024-12-06T01:54:16,859 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T01:54:16,859 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733450056859"}]},"ts":"1733450056859"}
2024-12-06T01:54:16,861 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-06T01:54:16,862 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T01:54:16,864 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 776 msec
2024-12-06T01:54:17,184 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:17,222 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:18,185 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:18,223 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:18,907 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:54:18,908 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,908 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,908 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,927 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,927 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,928 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:18,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:54:19,186 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:19,223 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:20,186 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:20,224 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:21,187 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:21,225 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:21,406 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T01:54:21,407 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-06T01:54:22,188 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:22,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:23,189 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:23,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:23,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-06T01:54:23,405 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-06T01:54:23,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:54:23,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-06T01:54:23,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-06T01:54:23,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-06T01:54:23,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:23,406 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-06T01:54:24,189 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:24,227 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:25,190 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:25,228 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:54:26,129 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T01:54:26,129 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-06T01:54:26,132 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:26,132 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079., hostname=00c764f7c3aa,42833,1733450055147, seqNum=2]
2024-12-06T01:54:26,142 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:26,148 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T01:54:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-06T01:54:26,150 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T01:54:26,151 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T01:54:26,191 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:26,229 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:26,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-06T01:54:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:26,312 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4ba57513049c73ee0fe02636937e1079 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T01:54:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/3da0e18f8df24210922ca9f2457dd57b is 1080, key is row0001/info:/1733450066136/Put/seqid=0
2024-12-06T01:54:26,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741837_1013 (size=6033)
2024-12-06T01:54:26,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741837_1013 (size=6033)
2024-12-06T01:54:26,335 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/3da0e18f8df24210922ca9f2457dd57b
2024-12-06T01:54:26,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/3da0e18f8df24210922ca9f2457dd57b as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3da0e18f8df24210922ca9f2457dd57b
2024-12-06T01:54:26,347 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3da0e18f8df24210922ca9f2457dd57b, entries=1, sequenceid=5, filesize=5.9 K
2024-12-06T01:54:26,348 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 4ba57513049c73ee0fe02636937e1079 in 36ms, sequenceid=5, compaction requested=false
2024-12-06T01:54:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4ba57513049c73ee0fe02636937e1079:

2024-12-06T01:54:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-06T01:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-06T01:54:26,356 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-06T01:54:26,356 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 202 msec
2024-12-06T01:54:26,359 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 213 msec
2024-12-06T01:54:27,191 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:27,229 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:28,192 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:28,230 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:29,193 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:29,231 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:30,194 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:30,231 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:31,194 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:31,232 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:32,195 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:32,233 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:33,196 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:33,233 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:34,197 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:34,234 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:35,197 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:35,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:36,198 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-06T01:54:36,219 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T01:54:36,223 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T01:54:36,225 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T01:54:36,226 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T01:54:36,226 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T01:54:36,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:36,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-06T01:54:36,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:36,380 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4ba57513049c73ee0fe02636937e1079 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T01:54:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/3beb9528ca264868ab89464f9db73b9a is 1080, key is row0002/info:/1733450076221/Put/seqid=0
2024-12-06T01:54:36,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741838_1014 (size=6033)
2024-12-06T01:54:36,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741838_1014 (size=6033)
2024-12-06T01:54:36,398 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/3beb9528ca264868ab89464f9db73b9a
2024-12-06T01:54:36,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/3beb9528ca264868ab89464f9db73b9a as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3beb9528ca264868ab89464f9db73b9a
2024-12-06T01:54:36,411 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3beb9528ca264868ab89464f9db73b9a, entries=1, sequenceid=9, filesize=5.9 K
2024-12-06T01:54:36,412 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 4ba57513049c73ee0fe02636937e1079 in 32ms, sequenceid=9, compaction requested=false
2024-12-06T01:54:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4ba57513049c73ee0fe02636937e1079:

2024-12-06T01:54:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-06T01:54:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-06T01:54:36,417 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-06T01:54:36,417 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-12-06T01:54:36,420 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 195 msec
2024-12-06T01:54:37,198 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:37,236 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:38,199 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:38,236 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:39,200 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:39,237 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:40,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:40,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140 after 68053ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor205.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:54:40,238 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:40,238 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta after 68046ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor205.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-06T01:54:41,202 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:41,238 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:42,202 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:42,239 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:43,203 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:43,240 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:44,204 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:44,240 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:45,077 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:54:45,204 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:45,241 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:46,205 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:46,242 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-06T01:54:46,309 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T01:54:46,312 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42833%2C1733450055147.1733450086312
2024-12-06T01:54:46,319 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:46,319 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:46,319 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:46,319 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:46,320 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:46,320 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450055560 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450086312
2024-12-06T01:54:46,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741833_1009 (size=5546)
2024-12-06T01:54:46,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741833_1009 (size=5546)
2024-12-06T01:54:46,329 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46711:46711),(127.0.0.1/127.0.0.1:45839:45839)]
2024-12-06T01:54:46,330 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T01:54:46,333 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T01:54:46,334 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T01:54:46,334 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T01:54:46,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-06T01:54:46,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:46,489 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4ba57513049c73ee0fe02636937e1079 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T01:54:46,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/e28f7b51a7244502be499d71d269f241 is 1080, key is row0003/info:/1733450086310/Put/seqid=0
2024-12-06T01:54:46,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741840_1016 (size=6033)
2024-12-06T01:54:46,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741840_1016 (size=6033)
2024-12-06T01:54:46,512 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/e28f7b51a7244502be499d71d269f241
2024-12-06T01:54:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/e28f7b51a7244502be499d71d269f241 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/e28f7b51a7244502be499d71d269f241
2024-12-06T01:54:46,526 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/e28f7b51a7244502be499d71d269f241, entries=1, sequenceid=13, filesize=5.9 K
2024-12-06T01:54:46,527 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 4ba57513049c73ee0fe02636937e1079 in 38ms, sequenceid=13, compaction requested=true
2024-12-06T01:54:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4ba57513049c73ee0fe02636937e1079:

2024-12-06T01:54:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-06T01:54:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-06T01:54:46,532 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-06T01:54:46,533 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 196 msec
2024-12-06T01:54:46,536 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 204 msec
2024-12-06T01:54:47,206 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:47,242 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:48,206 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:48,243 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:49,207 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:49,244 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:50,208 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:50,244 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:51,209 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:51,245 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:52,209 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:52,246 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:53,210 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:53,246 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:54,211 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:54,247 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:55,211 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:55,248 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:56,212 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:56,248 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-06T01:54:56,399 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T01:54:56,399 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:54:56,401 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:54:56,401 DEBUG [Time-limited test {}] regionserver.HStore(1541): 4ba57513049c73ee0fe02636937e1079/info is initiating minor compaction (all files)
2024-12-06T01:54:56,401 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:54:56,401 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:54:56,401 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 4ba57513049c73ee0fe02636937e1079/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:56,402 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3da0e18f8df24210922ca9f2457dd57b, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3beb9528ca264868ab89464f9db73b9a, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/e28f7b51a7244502be499d71d269f241] into tmpdir=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp, totalSize=17.7 K
2024-12-06T01:54:56,402 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3da0e18f8df24210922ca9f2457dd57b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733450066136
2024-12-06T01:54:56,403 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3beb9528ca264868ab89464f9db73b9a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733450076221
2024-12-06T01:54:56,403 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e28f7b51a7244502be499d71d269f241, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733450086310
2024-12-06T01:54:56,416 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 4ba57513049c73ee0fe02636937e1079#info#compaction#44 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:54:56,416 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/cf9a10f44f7e4f75aebb63f10a6da6d8 is 1080, key is row0001/info:/1733450066136/Put/seqid=0
2024-12-06T01:54:56,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741841_1017 (size=8296)
2024-12-06T01:54:56,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741841_1017 (size=8296)
2024-12-06T01:54:56,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/cf9a10f44f7e4f75aebb63f10a6da6d8 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/cf9a10f44f7e4f75aebb63f10a6da6d8
2024-12-06T01:54:56,437 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4ba57513049c73ee0fe02636937e1079/info of 4ba57513049c73ee0fe02636937e1079 into cf9a10f44f7e4f75aebb63f10a6da6d8(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:54:56,437 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 4ba57513049c73ee0fe02636937e1079:

2024-12-06T01:54:56,439 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42833%2C1733450055147.1733450096439
2024-12-06T01:54:56,446 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:56,446 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:56,446 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:56,446 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:56,446 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:54:56,446 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450086312 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450096439
2024-12-06T01:54:56,447 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46711:46711),(127.0.0.1/127.0.0.1:45839:45839)]
2024-12-06T01:54:56,447 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450086312 is not closed yet, will try archiving it next time
2024-12-06T01:54:56,448 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741839_1015 (size=2520)
2024-12-06T01:54:56,448 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741839_1015 (size=2520)
2024-12-06T01:54:56,450 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450055560 to hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/oldWALs/00c764f7c3aa%2C42833%2C1733450055147.1733450055560
2024-12-06T01:54:56,451 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:54:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T01:54:56,454 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-06T01:54:56,456 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-06T01:54:56,456 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-06T01:54:56,495 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T01:54:56,495 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T01:54:56,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42833 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-06T01:54:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:56,610 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4ba57513049c73ee0fe02636937e1079 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T01:54:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/044042538c224ca2a9ad8a3be411ad32 is 1080, key is row0000/info:/1733450096438/Put/seqid=0
2024-12-06T01:54:56,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741843_1019 (size=6033)
2024-12-06T01:54:56,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741843_1019 (size=6033)
2024-12-06T01:54:56,622 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/044042538c224ca2a9ad8a3be411ad32
2024-12-06T01:54:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/044042538c224ca2a9ad8a3be411ad32 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/044042538c224ca2a9ad8a3be411ad32
2024-12-06T01:54:56,635 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/044042538c224ca2a9ad8a3be411ad32, entries=1, sequenceid=18, filesize=5.9 K
2024-12-06T01:54:56,636 INFO  [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 4ba57513049c73ee0fe02636937e1079 in 26ms, sequenceid=18, compaction requested=false
2024-12-06T01:54:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4ba57513049c73ee0fe02636937e1079:

2024-12-06T01:54:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:54:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-06T01:54:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-06T01:54:56,641 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-06T01:54:56,641 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec
2024-12-06T01:54:56,644 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 191 msec
2024-12-06T01:54:57,213 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:57,249 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:58,214 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:58,250 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:59,214 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:54:59,250 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:00,215 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:00,251 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:01,216 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:01,251 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:01,835 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 4ba57513049c73ee0fe02636937e1079, had cached 0 bytes from a total of 14329
2024-12-06T01:55:02,216 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:02,252 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:03,217 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:03,252 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:04,218 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:04,253 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:05,218 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:05,254 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:06,219 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:06,254 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34381 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-06T01:55:06,560 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-06T01:55:06,565 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42833%2C1733450055147.1733450106565
2024-12-06T01:55:06,585 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:06,585 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:06,585 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:06,585 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:06,586 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:06,586 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450096439 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450106565
2024-12-06T01:55:06,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741842_1018 (size=2026)
2024-12-06T01:55:06,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741842_1018 (size=2026)
2024-12-06T01:55:06,589 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/WALs/00c764f7c3aa,42833,1733450055147/00c764f7c3aa%2C42833%2C1733450055147.1733450086312 to hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/oldWALs/00c764f7c3aa%2C42833%2C1733450055147.1733450086312
2024-12-06T01:55:06,612 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46711:46711),(127.0.0.1/127.0.0.1:45839:45839)]
2024-12-06T01:55:06,612 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:55:06,612 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:55:06,613 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:55:06,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:55:06,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:55:06,613 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:55:06,613 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=680476904, stopped=false
2024-12-06T01:55:06,613 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,34381,1733450055095
2024-12-06T01:55:06,614 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:55:06,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:55:06,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:06,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:55:06,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:06,616 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:55:06,616 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:55:06,616 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:55:06,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:55:06,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:55:06,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:55:06,619 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,42833,1733450055147' *****
2024-12-06T01:55:06,619 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:55:06,619 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:55:06,619 INFO  [RS:0;00c764f7c3aa:42833 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:55:06,619 INFO  [RS:0;00c764f7c3aa:42833 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:55:06,619 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:55:06,619 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:55:06,629 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,42833,1733450055147
2024-12-06T01:55:06,629 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:55:06,629 INFO  [RS:0;00c764f7c3aa:42833 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:42833.
2024-12-06T01:55:06,630 DEBUG [RS:0;00c764f7c3aa:42833 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:55:06,630 DEBUG [RS:0;00c764f7c3aa:42833 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:55:06,630 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:55:06,630 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:55:06,630 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:55:06,630 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:55:06,632 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ba57513049c73ee0fe02636937e1079, disabling compactions & flushes
2024-12-06T01:55:06,632 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-06T01:55:06,632 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1325): Online Regions={4ba57513049c73ee0fe02636937e1079=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079., 1588230740=hbase:meta,,1.1588230740}
2024-12-06T01:55:06,632 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4ba57513049c73ee0fe02636937e1079
2024-12-06T01:55:06,633 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079. after waiting 0 ms
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:55:06,633 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:55:06,633 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:55:06,633 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4ba57513049c73ee0fe02636937e1079 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-06T01:55:06,633 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-06T01:55:06,654 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/0b1fa3da81c1432aad0f84db50b1be25 is 1080, key is row0001/info:/1733450106562/Put/seqid=0
2024-12-06T01:55:06,660 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/info/78c113531643437785e94fa02bb52c97 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079./info:regioninfo/1733450056849/Put/seqid=0
2024-12-06T01:55:06,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741845_1021 (size=6033)
2024-12-06T01:55:06,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741846_1022 (size=7308)
2024-12-06T01:55:06,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741846_1022 (size=7308)
2024-12-06T01:55:06,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741845_1021 (size=6033)
2024-12-06T01:55:06,686 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/0b1fa3da81c1432aad0f84db50b1be25
2024-12-06T01:55:06,698 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/.tmp/info/0b1fa3da81c1432aad0f84db50b1be25 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/0b1fa3da81c1432aad0f84db50b1be25
2024-12-06T01:55:06,706 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/0b1fa3da81c1432aad0f84db50b1be25, entries=1, sequenceid=22, filesize=5.9 K
2024-12-06T01:55:06,708 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 4ba57513049c73ee0fe02636937e1079 in 75ms, sequenceid=22, compaction requested=true
2024-12-06T01:55:06,708 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3da0e18f8df24210922ca9f2457dd57b, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3beb9528ca264868ab89464f9db73b9a, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/e28f7b51a7244502be499d71d269f241] to archive
2024-12-06T01:55:06,709 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T01:55:06,712 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3da0e18f8df24210922ca9f2457dd57b to hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3da0e18f8df24210922ca9f2457dd57b
2024-12-06T01:55:06,715 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3beb9528ca264868ab89464f9db73b9a to hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/3beb9528ca264868ab89464f9db73b9a
2024-12-06T01:55:06,717 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/e28f7b51a7244502be499d71d269f241 to hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/info/e28f7b51a7244502be499d71d269f241
2024-12-06T01:55:06,718 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=00c764f7c3aa:34381 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-06T01:55:06,718 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [3da0e18f8df24210922ca9f2457dd57b=6033, 3beb9528ca264868ab89464f9db73b9a=6033, e28f7b51a7244502be499d71d269f241=6033]
2024-12-06T01:55:06,728 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/4ba57513049c73ee0fe02636937e1079/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-06T01:55:06,729 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:55:06,730 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ba57513049c73ee0fe02636937e1079:
Waiting for close lock at 1733450106632Running coprocessor pre-close hooks at 1733450106632Disabling compacts and flushes for region at 1733450106632Disabling writes for close at 1733450106633 (+1 ms)Obtaining lock to block concurrent updates at 1733450106633Preparing flush snapshotting stores in 4ba57513049c73ee0fe02636937e1079 at 1733450106633Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733450106634 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079. at 1733450106642 (+8 ms)Flushing 4ba57513049c73ee0fe02636937e1079/info: creating writer at 1733450106642Flushing 4ba57513049c73ee0fe02636937e1079/info: appending metadata at 1733450106653 (+11 ms)Flushing 4ba57513049c73ee0fe02636937e1079/info: closing flushed file at 1733450106654 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cfb7b9d: reopening flushed file at 1733450106696 (+42 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 4ba57513049c73ee0fe02636937e1079 in 75ms, sequenceid=22, compaction requested=true at 1733450106708 (+12 ms)Writing region close event to WAL at 1733450106719 (+11 ms)Running coprocessor post-close hooks at 1733450106729 (+10 ms)Closed at 1733450106729
2024-12-06T01:55:06,730 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733450056085.4ba57513049c73ee0fe02636937e1079.
2024-12-06T01:55:06,833 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:55:07,033 DEBUG [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:55:07,083 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/info/78c113531643437785e94fa02bb52c97
2024-12-06T01:55:07,106 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/ns/e3f7a70b27f94da2b5071b430d2d873f is 43, key is default/ns:d/1733450055964/Put/seqid=0
2024-12-06T01:55:07,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741847_1023 (size=5153)
2024-12-06T01:55:07,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741847_1023 (size=5153)
2024-12-06T01:55:07,112 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/ns/e3f7a70b27f94da2b5071b430d2d873f
2024-12-06T01:55:07,136 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/table/a1f20732ecdf4aada3eafc4f7e3e75d3 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733450056859/Put/seqid=0
2024-12-06T01:55:07,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741848_1024 (size=5508)
2024-12-06T01:55:07,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741848_1024 (size=5508)
2024-12-06T01:55:07,156 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/table/a1f20732ecdf4aada3eafc4f7e3e75d3
2024-12-06T01:55:07,163 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/info/78c113531643437785e94fa02bb52c97 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/info/78c113531643437785e94fa02bb52c97
2024-12-06T01:55:07,174 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/info/78c113531643437785e94fa02bb52c97, entries=10, sequenceid=11, filesize=7.1 K
2024-12-06T01:55:07,176 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/ns/e3f7a70b27f94da2b5071b430d2d873f as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/ns/e3f7a70b27f94da2b5071b430d2d873f
2024-12-06T01:55:07,182 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/ns/e3f7a70b27f94da2b5071b430d2d873f, entries=2, sequenceid=11, filesize=5.0 K
2024-12-06T01:55:07,183 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/.tmp/table/a1f20732ecdf4aada3eafc4f7e3e75d3 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/table/a1f20732ecdf4aada3eafc4f7e3e75d3
2024-12-06T01:55:07,189 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/table/a1f20732ecdf4aada3eafc4f7e3e75d3, entries=2, sequenceid=11, filesize=5.4 K
2024-12-06T01:55:07,191 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 557ms, sequenceid=11, compaction requested=false
2024-12-06T01:55:07,196 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-06T01:55:07,197 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:55:07,197 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:55:07,197 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450106633Running coprocessor pre-close hooks at 1733450106633Disabling compacts and flushes for region at 1733450106633Disabling writes for close at 1733450106633Obtaining lock to block concurrent updates at 1733450106633Preparing flush snapshotting stores in 1588230740 at 1733450106633Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733450106634 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733450106635 (+1 ms)Flushing 1588230740/info: creating writer at 1733450106635Flushing 1588230740/info: appending metadata at 1733450106660 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733450106660Flushing 1588230740/ns: creating writer at 1733450107090 (+430 ms)Flushing 1588230740/ns: appending metadata at 1733450107105 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733450107105Flushing 1588230740/table: creating writer at 1733450107118 (+13 ms)Flushing 1588230740/table: appending metadata at 1733450107135 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733450107135Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@305890b3: reopening flushed file at 1733450107162 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@800cf9a: reopening flushed file at 1733450107175 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a875ca6: reopening flushed file at 1733450107182 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 557ms, sequenceid=11, compaction requested=false at 1733450107191 (+9 ms)Writing region close event to WAL at 1733450107192 (+1 ms)Running coprocessor post-close hooks at 1733450107197 (+5 ms)Closed at 1733450107197
2024-12-06T01:55:07,197 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:55:07,219 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:07,233 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,42833,1733450055147; all regions closed.
2024-12-06T01:55:07,234 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,234 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,234 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,234 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,234 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741834_1010 (size=3306)
2024-12-06T01:55:07,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741834_1010 (size=3306)
2024-12-06T01:55:07,239 DEBUG [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/oldWALs
2024-12-06T01:55:07,239 INFO  [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C42833%2C1733450055147.meta:.meta(num 1733450055924)
2024-12-06T01:55:07,240 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,240 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,240 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,240 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,240 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741844_1020 (size=1252)
2024-12-06T01:55:07,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741844_1020 (size=1252)
2024-12-06T01:55:07,246 DEBUG [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/oldWALs
2024-12-06T01:55:07,246 INFO  [RS:0;00c764f7c3aa:42833 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C42833%2C1733450055147:(num 1733450106565)
2024-12-06T01:55:07,246 DEBUG [RS:0;00c764f7c3aa:42833 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:55:07,246 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:55:07,247 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:55:07,247 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T01:55:07,247 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:55:07,247 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:55:07,247 INFO  [RS:0;00c764f7c3aa:42833 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42833
2024-12-06T01:55:07,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:55:07,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,42833,1733450055147
2024-12-06T01:55:07,250 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:55:07,251 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,42833,1733450055147]
2024-12-06T01:55:07,253 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,42833,1733450055147 already deleted, retry=false
2024-12-06T01:55:07,253 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,42833,1733450055147 expired; onlineServers=0
2024-12-06T01:55:07,253 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,34381,1733450055095' *****
2024-12-06T01:55:07,253 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:55:07,253 INFO  [M:0;00c764f7c3aa:34381 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:55:07,253 INFO  [M:0;00c764f7c3aa:34381 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:55:07,253 DEBUG [M:0;00c764f7c3aa:34381 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:55:07,253 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:55:07,253 DEBUG [M:0;00c764f7c3aa:34381 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:55:07,253 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450055324 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450055324,5,FailOnTimeoutGroup]
2024-12-06T01:55:07,253 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450055324 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450055324,5,FailOnTimeoutGroup]
2024-12-06T01:55:07,253 INFO  [M:0;00c764f7c3aa:34381 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:55:07,253 INFO  [M:0;00c764f7c3aa:34381 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:55:07,253 DEBUG [M:0;00c764f7c3aa:34381 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:55:07,253 INFO  [M:0;00c764f7c3aa:34381 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:55:07,254 INFO  [M:0;00c764f7c3aa:34381 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:55:07,254 INFO  [M:0;00c764f7c3aa:34381 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:55:07,254 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:55:07,255 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:07,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:55:07,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:07,256 DEBUG [M:0;00c764f7c3aa:34381 {}] zookeeper.ZKUtil(347): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:55:07,256 WARN  [M:0;00c764f7c3aa:34381 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:55:07,256 INFO  [M:0;00c764f7c3aa:34381 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/.lastflushedseqids
2024-12-06T01:55:07,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741849_1025 (size=130)
2024-12-06T01:55:07,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741849_1025 (size=130)
2024-12-06T01:55:07,272 INFO  [M:0;00c764f7c3aa:34381 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:55:07,272 INFO  [M:0;00c764f7c3aa:34381 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:55:07,272 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:55:07,272 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:07,272 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:07,272 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:55:07,272 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:07,272 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.57 KB heapSize=54.96 KB
2024-12-06T01:55:07,289 DEBUG [M:0;00c764f7c3aa:34381 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e698645a19340c8b51b09a56da82500 is 82, key is hbase:meta,,1/info:regioninfo/1733450055949/Put/seqid=0
2024-12-06T01:55:07,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741850_1026 (size=5672)
2024-12-06T01:55:07,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741850_1026 (size=5672)
2024-12-06T01:55:07,319 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e698645a19340c8b51b09a56da82500
2024-12-06T01:55:07,352 INFO  [RS:0;00c764f7c3aa:42833 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:55:07,353 INFO  [RS:0;00c764f7c3aa:42833 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,42833,1733450055147; zookeeper connection closed.
2024-12-06T01:55:07,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:55:07,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42833-0x1006508768d0001, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:55:07,355 DEBUG [M:0;00c764f7c3aa:34381 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e90463c88eb482b9fa0e89e1f3c22fe is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733450056863/Put/seqid=0
2024-12-06T01:55:07,359 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54b18ed4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54b18ed4
2024-12-06T01:55:07,360 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T01:55:07,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741851_1027 (size=7820)
2024-12-06T01:55:07,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741851_1027 (size=7820)
2024-12-06T01:55:07,379 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e90463c88eb482b9fa0e89e1f3c22fe
2024-12-06T01:55:07,384 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e90463c88eb482b9fa0e89e1f3c22fe
2024-12-06T01:55:07,410 DEBUG [M:0;00c764f7c3aa:34381 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/764f54faea73471c95e01d3887c47141 is 69, key is 00c764f7c3aa,42833,1733450055147/rs:state/1733450055398/Put/seqid=0
2024-12-06T01:55:07,415 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:55:07,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741852_1028 (size=5156)
2024-12-06T01:55:07,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741852_1028 (size=5156)
2024-12-06T01:55:07,427 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/764f54faea73471c95e01d3887c47141
2024-12-06T01:55:07,451 DEBUG [M:0;00c764f7c3aa:34381 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f30dadb599834caab1135d0d898e1129 is 52, key is load_balancer_on/state:d/1733450056081/Put/seqid=0
2024-12-06T01:55:07,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741853_1029 (size=5056)
2024-12-06T01:55:07,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741853_1029 (size=5056)
2024-12-06T01:55:07,458 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f30dadb599834caab1135d0d898e1129
2024-12-06T01:55:07,465 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e698645a19340c8b51b09a56da82500 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e698645a19340c8b51b09a56da82500
2024-12-06T01:55:07,472 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e698645a19340c8b51b09a56da82500, entries=8, sequenceid=121, filesize=5.5 K
2024-12-06T01:55:07,473 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e90463c88eb482b9fa0e89e1f3c22fe as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0e90463c88eb482b9fa0e89e1f3c22fe
2024-12-06T01:55:07,483 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e90463c88eb482b9fa0e89e1f3c22fe
2024-12-06T01:55:07,483 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0e90463c88eb482b9fa0e89e1f3c22fe, entries=14, sequenceid=121, filesize=7.6 K
2024-12-06T01:55:07,485 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/764f54faea73471c95e01d3887c47141 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/764f54faea73471c95e01d3887c47141
2024-12-06T01:55:07,490 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/764f54faea73471c95e01d3887c47141, entries=1, sequenceid=121, filesize=5.0 K
2024-12-06T01:55:07,491 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f30dadb599834caab1135d0d898e1129 as hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f30dadb599834caab1135d0d898e1129
2024-12-06T01:55:07,498 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38289/user/jenkins/test-data/05b246b6-8414-dbf1-b35f-83bfa5506297/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f30dadb599834caab1135d0d898e1129, entries=1, sequenceid=121, filesize=4.9 K
2024-12-06T01:55:07,500 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 228ms, sequenceid=121, compaction requested=false
2024-12-06T01:55:07,505 INFO  [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:07,505 DEBUG [M:0;00c764f7c3aa:34381 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450107272Disabling compacts and flushes for region at 1733450107272Disabling writes for close at 1733450107272Obtaining lock to block concurrent updates at 1733450107272Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733450107272Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44611, getHeapSize=56216, getOffHeapSize=0, getCellsCount=140 at 1733450107273 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733450107273Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733450107273Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733450107288 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733450107288Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733450107326 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733450107354 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733450107354Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733450107384 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733450107409 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733450107409Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733450107434 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733450107451 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733450107451Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18b95baf: reopening flushed file at 1733450107464 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71d7d08f: reopening flushed file at 1733450107472 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b86576c: reopening flushed file at 1733450107483 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bf51dee: reopening flushed file at 1733450107491 (+8 ms)Finished flush of dataSize ~43.57 KB/44611, heapSize ~54.90 KB/56216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 228ms, sequenceid=121, compaction requested=false at 1733450107500 (+9 ms)Writing region close event to WAL at 1733450107505 (+5 ms)Closed at 1733450107505
2024-12-06T01:55:07,506 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,506 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,506 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,506 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,506 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:55:07,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45057 is added to blk_1073741830_1006 (size=53008)
2024-12-06T01:55:07,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39353 is added to blk_1073741830_1006 (size=53008)
2024-12-06T01:55:07,911 INFO  [M:0;00c764f7c3aa:34381 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:55:07,911 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:55:07,911 INFO  [M:0;00c764f7c3aa:34381 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34381
2024-12-06T01:55:07,911 INFO  [M:0;00c764f7c3aa:34381 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:55:08,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:55:08,014 INFO  [M:0;00c764f7c3aa:34381 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:55:08,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34381-0x1006508768d0000, quorum=127.0.0.1:61114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:55:08,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1780e3e3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:55:08,019 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c93132a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:55:08,019 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:55:08,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@392e4f56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:55:08,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@667e76c4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir/,STOPPED}
2024-12-06T01:55:08,023 WARN  [BP-2074420978-172.17.0.2-1733450054407 heartbeating to localhost/127.0.0.1:38289 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:55:08,023 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:55:08,023 WARN  [BP-2074420978-172.17.0.2-1733450054407 heartbeating to localhost/127.0.0.1:38289 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2074420978-172.17.0.2-1733450054407 (Datanode Uuid 38721538-9187-4051-a36d-31cce0b7b9e9) service to localhost/127.0.0.1:38289
2024-12-06T01:55:08,023 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:55:08,023 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data3/current/BP-2074420978-172.17.0.2-1733450054407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:55:08,024 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data4/current/BP-2074420978-172.17.0.2-1733450054407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:55:08,024 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:55:08,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@63c886b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:55:08,027 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e71ae2f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:55:08,027 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:55:08,027 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24056a85{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:55:08,027 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@418725d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir/,STOPPED}
2024-12-06T01:55:08,029 WARN  [BP-2074420978-172.17.0.2-1733450054407 heartbeating to localhost/127.0.0.1:38289 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:55:08,029 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:55:08,029 WARN  [BP-2074420978-172.17.0.2-1733450054407 heartbeating to localhost/127.0.0.1:38289 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2074420978-172.17.0.2-1733450054407 (Datanode Uuid 1cb5fa48-0475-4200-b9b1-f8a963cd20bf) service to localhost/127.0.0.1:38289
2024-12-06T01:55:08,029 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:55:08,030 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data1/current/BP-2074420978-172.17.0.2-1733450054407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:55:08,030 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/cluster_488132fd-b476-3a5f-257c-85a210c78c9a/data/data2/current/BP-2074420978-172.17.0.2-1733450054407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:55:08,031 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:55:08,040 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ee3adbc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:55:08,041 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1210b54d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:55:08,041 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:55:08,041 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1df61fd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:55:08,041 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67166726{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir/,STOPPED}
2024-12-06T01:55:08,050 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:55:08,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:55:08,081 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=204 (was 179)
Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38289
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:38289 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:38289 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:38289 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38289
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38289
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38289
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:38289
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=485 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 323), ProcessCount=11 (was 11), AvailableMemoryMB=9193 (was 9712)
2024-12-06T01:55:08,090 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=204, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=9193
2024-12-06T01:55:08,090 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:55:08,090 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.log.dir so I do NOT create it in target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30
2024-12-06T01:55:08,090 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1e9e49ec-5e6a-0775-d152-b4641d6534e8/hadoop.tmp.dir so I do NOT create it in target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30
2024-12-06T01:55:08,090 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1, deleteOnExit=true
2024-12-06T01:55:08,090 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/test.cache.data in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:55:08,091 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:55:08,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:55:08,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:55:08,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:55:08,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:55:08,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:55:08,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:55:08,092 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:55:08,106 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:55:08,175 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:55:08,179 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:55:08,180 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:55:08,180 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:55:08,180 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:55:08,181 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:55:08,181 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5eef5514{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:55:08,181 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10fad9f2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:55:08,220 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:08,256 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:08,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b14e4d9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/java.io.tmpdir/jetty-localhost-44545-hadoop-hdfs-3_4_1-tests_jar-_-any-13553882862338814739/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:55:08,325 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2424cbbd{HTTP/1.1, (http/1.1)}{localhost:44545}
2024-12-06T01:55:08,325 INFO  [Time-limited test {}] server.Server(415): Started @241593ms
2024-12-06T01:55:08,339 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:55:08,421 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:55:08,425 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:55:08,426 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:55:08,426 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:55:08,426 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:55:08,427 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36ab964c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:55:08,427 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@646e9349{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:55:08,594 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b03ccf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/java.io.tmpdir/jetty-localhost-43979-hadoop-hdfs-3_4_1-tests_jar-_-any-17259549509869092497/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:55:08,594 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bc201cc{HTTP/1.1, (http/1.1)}{localhost:43979}
2024-12-06T01:55:08,594 INFO  [Time-limited test {}] server.Server(415): Started @241862ms
2024-12-06T01:55:08,596 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:55:08,666 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:55:08,670 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:55:08,673 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:55:08,673 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:55:08,673 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:55:08,676 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@140f1428{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:55:08,677 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6fea856d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:55:08,722 WARN  [Thread-1959 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data1/current/BP-842780030-172.17.0.2-1733450108112/current, will proceed with Du for space computation calculation, 
2024-12-06T01:55:08,722 WARN  [Thread-1960 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data2/current/BP-842780030-172.17.0.2-1733450108112/current, will proceed with Du for space computation calculation, 
2024-12-06T01:55:08,747 WARN  [Thread-1938 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:55:08,750 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84568508bc5f7a8d with lease ID 0x509074146420c4e: Processing first storage report for DS-c4f12e5c-3706-4f56-ae3c-c262a57b2f2a from datanode DatanodeRegistration(127.0.0.1:42917, datanodeUuid=4af9ca99-38e6-42e0-aef4-c33476dbd857, infoPort=33787, infoSecurePort=0, ipcPort=37241, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112)
2024-12-06T01:55:08,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84568508bc5f7a8d with lease ID 0x509074146420c4e: from storage DS-c4f12e5c-3706-4f56-ae3c-c262a57b2f2a node DatanodeRegistration(127.0.0.1:42917, datanodeUuid=4af9ca99-38e6-42e0-aef4-c33476dbd857, infoPort=33787, infoSecurePort=0, ipcPort=37241, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:55:08,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84568508bc5f7a8d with lease ID 0x509074146420c4e: Processing first storage report for DS-34133a70-3a7a-4460-bbf7-b2e012cecd14 from datanode DatanodeRegistration(127.0.0.1:42917, datanodeUuid=4af9ca99-38e6-42e0-aef4-c33476dbd857, infoPort=33787, infoSecurePort=0, ipcPort=37241, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112)
2024-12-06T01:55:08,751 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84568508bc5f7a8d with lease ID 0x509074146420c4e: from storage DS-34133a70-3a7a-4460-bbf7-b2e012cecd14 node DatanodeRegistration(127.0.0.1:42917, datanodeUuid=4af9ca99-38e6-42e0-aef4-c33476dbd857, infoPort=33787, infoSecurePort=0, ipcPort=37241, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:55:08,813 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@62812bcb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/java.io.tmpdir/jetty-localhost-38471-hadoop-hdfs-3_4_1-tests_jar-_-any-15559900985899048673/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:55:08,813 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5fe69f2e{HTTP/1.1, (http/1.1)}{localhost:38471}
2024-12-06T01:55:08,813 INFO  [Time-limited test {}] server.Server(415): Started @242081ms
2024-12-06T01:55:08,814 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:55:08,913 WARN  [Thread-1985 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data3/current/BP-842780030-172.17.0.2-1733450108112/current, will proceed with Du for space computation calculation, 
2024-12-06T01:55:08,914 WARN  [Thread-1986 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data4/current/BP-842780030-172.17.0.2-1733450108112/current, will proceed with Du for space computation calculation, 
2024-12-06T01:55:08,936 WARN  [Thread-1974 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:55:08,940 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0ba9a61919de9 with lease ID 0x509074146420c4f: Processing first storage report for DS-d41f8c5b-df7c-4d94-bf11-b3e8c55cc10f from datanode DatanodeRegistration(127.0.0.1:32953, datanodeUuid=33f0282b-1ef2-4dec-bb90-c170b358e5e9, infoPort=35651, infoSecurePort=0, ipcPort=36903, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112)
2024-12-06T01:55:08,940 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0ba9a61919de9 with lease ID 0x509074146420c4f: from storage DS-d41f8c5b-df7c-4d94-bf11-b3e8c55cc10f node DatanodeRegistration(127.0.0.1:32953, datanodeUuid=33f0282b-1ef2-4dec-bb90-c170b358e5e9, infoPort=35651, infoSecurePort=0, ipcPort=36903, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:55:08,940 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0ba9a61919de9 with lease ID 0x509074146420c4f: Processing first storage report for DS-079c9858-f319-4bda-be6d-75dd0cdfba8e from datanode DatanodeRegistration(127.0.0.1:32953, datanodeUuid=33f0282b-1ef2-4dec-bb90-c170b358e5e9, infoPort=35651, infoSecurePort=0, ipcPort=36903, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112)
2024-12-06T01:55:08,940 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0ba9a61919de9 with lease ID 0x509074146420c4f: from storage DS-079c9858-f319-4bda-be6d-75dd0cdfba8e node DatanodeRegistration(127.0.0.1:32953, datanodeUuid=33f0282b-1ef2-4dec-bb90-c170b358e5e9, infoPort=35651, infoSecurePort=0, ipcPort=36903, storageInfo=lv=-57;cid=testClusterID;nsid=1961681828;c=1733450108112), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:55:08,942 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30
2024-12-06T01:55:08,945 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/zookeeper_0, clientPort=62741, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:55:08,946 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62741
2024-12-06T01:55:08,946 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:08,947 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:08,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:55:08,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:55:08,961 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db with version=8
2024-12-06T01:55:08,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:55:08,963 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:55:08,963 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:55:08,963 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:55:08,964 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:55:08,964 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:55:08,964 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:55:08,964 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:55:08,964 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:55:08,964 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36107
2024-12-06T01:55:08,966 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36107 connecting to ZooKeeper ensemble=127.0.0.1:62741
2024-12-06T01:55:08,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361070x0, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:55:08,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36107-0x100650948f60000 connected
2024-12-06T01:55:08,993 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:08,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:08,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:55:08,997 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db, hbase.cluster.distributed=false
2024-12-06T01:55:08,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:55:08,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36107
2024-12-06T01:55:08,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36107
2024-12-06T01:55:09,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36107
2024-12-06T01:55:09,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36107
2024-12-06T01:55:09,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36107
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:55:09,023 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:55:09,024 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:55:09,024 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45189
2024-12-06T01:55:09,026 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45189 connecting to ZooKeeper ensemble=127.0.0.1:62741
2024-12-06T01:55:09,027 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:09,029 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:09,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451890x0, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:55:09,034 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451890x0, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:55:09,034 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:55:09,035 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45189-0x100650948f60001 connected
2024-12-06T01:55:09,035 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:55:09,036 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:55:09,037 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:55:09,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45189
2024-12-06T01:55:09,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45189
2024-12-06T01:55:09,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45189
2024-12-06T01:55:09,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45189
2024-12-06T01:55:09,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45189
2024-12-06T01:55:09,054 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:36107
2024-12-06T01:55:09,054 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:55:09,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:55:09,056 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,059 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:55:09,059 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,36107,1733450108963 from backup master directory
2024-12-06T01:55:09,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:55:09,062 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:55:09,062 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:55:09,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:55:09,067 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/hbase.id] with ID: 1885f12a-8a09-4f80-aa0d-1cb6ac4c02b9
2024-12-06T01:55:09,067 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/.tmp/hbase.id
2024-12-06T01:55:09,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:55:09,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:55:09,075 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/.tmp/hbase.id]:[hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/hbase.id]
2024-12-06T01:55:09,092 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:09,093 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:55:09,094 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T01:55:09,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,103 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:55:09,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:55:09,104 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:55:09,105 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:55:09,105 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:55:09,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:55:09,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:55:09,119 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store
2024-12-06T01:55:09,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:55:09,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:55:09,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:09,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:55:09,135 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:09,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:09,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:55:09,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:09,135 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:55:09,135 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450109135Disabling compacts and flushes for region at 1733450109135Disabling writes for close at 1733450109135Writing region close event to WAL at 1733450109135Closed at 1733450109135
2024-12-06T01:55:09,136 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/.initializing
2024-12-06T01:55:09,136 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/WALs/00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,139 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C36107%2C1733450108963, suffix=, logDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/WALs/00c764f7c3aa,36107,1733450108963, archiveDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/oldWALs, maxLogs=10
2024-12-06T01:55:09,140 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C36107%2C1733450108963.1733450109140
2024-12-06T01:55:09,149 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/WALs/00c764f7c3aa,36107,1733450108963/00c764f7c3aa%2C36107%2C1733450108963.1733450109140
2024-12-06T01:55:09,152 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33787:33787),(127.0.0.1/127.0.0.1:35651:35651)]
2024-12-06T01:55:09,156 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:55:09,156 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:09,156 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,156 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,158 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:55:09,160 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:55:09,160 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,160 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,160 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:55:09,162 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:55:09,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,162 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:55:09,163 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:55:09,164 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:55:09,164 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,164 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:55:09,164 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:55:09,166 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:55:09,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,166 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:55:09,166 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,167 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,167 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,169 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,169 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,170 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:55:09,171 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:55:09,175 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:55:09,176 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=734609, jitterRate=-0.06589676439762115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:55:09,177 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733450109157Initializing all the Stores at 1733450109158 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109158Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450109158Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450109158Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450109158Cleaning up temporary data from old regions at 1733450109169 (+11 ms)Region opened successfully at 1733450109177 (+8 ms)
2024-12-06T01:55:09,177 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:55:09,181 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ad4166, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:55:09,182 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:55:09,182 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:55:09,182 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:55:09,182 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:55:09,183 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T01:55:09,183 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T01:55:09,183 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:55:09,186 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:55:09,187 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:55:09,189 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:55:09,189 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:55:09,190 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:55:09,192 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:55:09,192 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:55:09,193 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:55:09,194 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:55:09,195 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:55:09,196 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:55:09,198 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:55:09,200 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:55:09,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:55:09,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:55:09,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,203 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,36107,1733450108963, sessionid=0x100650948f60000, setting cluster-up flag (Was=false)
2024-12-06T01:55:09,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,211 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:55:09,212 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,221 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:09,224 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:55:09,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:09,228 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:55:09,232 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:55:09,232 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:55:09,232 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:55:09,232 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,36107,1733450108963
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:55:09,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,240 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:55:09,240 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:55:09,242 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,242 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:55:09,252 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733450139252
2024-12-06T01:55:09,252 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:55:09,253 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:55:09,253 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:55:09,253 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:55:09,253 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:55:09,253 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:55:09,253 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,255 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(746): ClusterId : 1885f12a-8a09-4f80-aa0d-1cb6ac4c02b9
2024-12-06T01:55:09,255 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:55:09,256 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:55:09,256 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:55:09,256 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:55:09,256 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:55:09,257 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:55:09,257 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:09,257 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:55:09,257 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:55:09,259 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:55:09,260 DEBUG [RS:0;00c764f7c3aa:45189 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2d9496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:55:09,263 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450109257,5,FailOnTimeoutGroup]
2024-12-06T01:55:09,267 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450109263,5,FailOnTimeoutGroup]
2024-12-06T01:55:09,267 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,267 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:55:09,267 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,267 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,274 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:55:09,274 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:55:09,276 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:55:09,276 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db
2024-12-06T01:55:09,286 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:45189
2024-12-06T01:55:09,286 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:55:09,286 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:55:09,286 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:55:09,288 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,36107,1733450108963 with port=45189, startcode=1733450109023
2024-12-06T01:55:09,288 DEBUG [RS:0;00c764f7c3aa:45189 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:55:09,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:55:09,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:55:09,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:09,291 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33131, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:55:09,292 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,292 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,293 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:55:09,295 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:55:09,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,295 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db
2024-12-06T01:55:09,295 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43255
2024-12-06T01:55:09,296 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:55:09,297 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,297 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:55:09,299 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:55:09,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:55:09,300 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,300 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:55:09,300 DEBUG [RS:0;00c764f7c3aa:45189 {}] zookeeper.ZKUtil(111): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,300 WARN  [RS:0;00c764f7c3aa:45189 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:55:09,300 INFO  [RS:0;00c764f7c3aa:45189 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:55:09,301 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,302 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:55:09,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,303 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,303 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:55:09,304 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:55:09,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,305 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:55:09,306 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,45189,1733450109023]
2024-12-06T01:55:09,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740
2024-12-06T01:55:09,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740
2024-12-06T01:55:09,307 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:55:09,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:55:09,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:55:09,309 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:55:09,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:55:09,312 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:55:09,312 INFO  [RS:0;00c764f7c3aa:45189 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:55:09,312 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,312 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:55:09,313 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:55:09,313 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=830650, jitterRate=0.05622692406177521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:55:09,313 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:55:09,313 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733450109290Initializing all the Stores at 1733450109293 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109293Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109293Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450109293Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109293Cleaning up temporary data from old regions at 1733450109308 (+15 ms)Region opened successfully at 1733450109314 (+6 ms)
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:55:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:55:09,314 DEBUG [RS:0;00c764f7c3aa:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:55:09,317 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:55:09,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450109314Disabling compacts and flushes for region at 1733450109314Disabling writes for close at 1733450109314Writing region close event to WAL at 1733450109317 (+3 ms)Closed at 1733450109317
2024-12-06T01:55:09,318 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:55:09,318 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:55:09,318 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:55:09,320 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:55:09,321 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:55:09,324 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,324 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,324 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,324 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,324 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,324 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,45189,1733450109023-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:55:09,346 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:55:09,346 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,45189,1733450109023-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,346 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,347 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.Replication(171): 00c764f7c3aa,45189,1733450109023 started
2024-12-06T01:55:09,368 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,368 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,45189,1733450109023, RpcServer on 00c764f7c3aa/172.17.0.2:45189, sessionid=0x100650948f60001
2024-12-06T01:55:09,369 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:55:09,369 DEBUG [RS:0;00c764f7c3aa:45189 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,369 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,45189,1733450109023'
2024-12-06T01:55:09,369 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:55:09,370 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:55:09,370 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:55:09,370 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:55:09,370 DEBUG [RS:0;00c764f7c3aa:45189 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,370 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,45189,1733450109023'
2024-12-06T01:55:09,370 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:55:09,371 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:55:09,371 DEBUG [RS:0;00c764f7c3aa:45189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:55:09,371 INFO  [RS:0;00c764f7c3aa:45189 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:55:09,371 INFO  [RS:0;00c764f7c3aa:45189 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:55:09,472 WARN  [00c764f7c3aa:36107 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:55:09,474 INFO  [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C45189%2C1733450109023, suffix=, logDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023, archiveDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/oldWALs, maxLogs=32
2024-12-06T01:55:09,474 INFO  [RS:0;00c764f7c3aa:45189 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45189%2C1733450109023.1733450109474
2024-12-06T01:55:09,483 INFO  [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450109474
2024-12-06T01:55:09,485 DEBUG [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33787:33787),(127.0.0.1/127.0.0.1:35651:35651)]
2024-12-06T01:55:09,722 DEBUG [00c764f7c3aa:36107 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:55:09,723 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,724 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,45189,1733450109023, state=OPENING
2024-12-06T01:55:09,726 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:55:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:55:09,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:55:09,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:55:09,729 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45189,1733450109023}]
2024-12-06T01:55:09,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:55:09,883 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:55:09,886 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:55:09,890 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:55:09,890 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:55:09,892 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C45189%2C1733450109023.meta, suffix=.meta, logDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023, archiveDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/oldWALs, maxLogs=32
2024-12-06T01:55:09,893 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45189%2C1733450109023.meta.1733450109893.meta
2024-12-06T01:55:09,920 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.meta.1733450109893.meta
2024-12-06T01:55:09,922 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33787:33787),(127.0.0.1/127.0.0.1:35651:35651)]
2024-12-06T01:55:09,927 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:55:09,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:55:09,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:55:09,928 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:55:09,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:55:09,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:09,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:55:09,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:55:09,930 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:55:09,931 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:55:09,931 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,931 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,931 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:55:09,932 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:55:09,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,933 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,933 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:55:09,934 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:55:09,934 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:55:09,935 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:55:09,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:09,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:55:09,936 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:55:09,937 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740
2024-12-06T01:55:09,938 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740
2024-12-06T01:55:09,940 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:55:09,940 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:55:09,941 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:55:09,942 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:55:09,944 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=851057, jitterRate=0.08217546343803406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:55:09,944 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:55:09,945 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733450109928Writing region info on filesystem at 1733450109928Initializing all the Stores at 1733450109929 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109929Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109930 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450109930Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450109930Cleaning up temporary data from old regions at 1733450109940 (+10 ms)Running coprocessor post-open hooks at 1733450109944 (+4 ms)Region opened successfully at 1733450109945 (+1 ms)
2024-12-06T01:55:09,950 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733450109883
2024-12-06T01:55:09,955 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:55:09,955 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:55:09,959 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,960 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,45189,1733450109023, state=OPEN
2024-12-06T01:55:09,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:55:09,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:55:09,965 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:09,965 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:55:09,965 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:55:09,968 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:55:09,968 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45189,1733450109023 in 236 msec
2024-12-06T01:55:09,970 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:55:09,971 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 650 msec
2024-12-06T01:55:09,971 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:55:09,971 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:55:09,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:55:09,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45189,1733450109023, seqNum=-1]
2024-12-06T01:55:09,973 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:55:09,975 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57055, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:55:09,981 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 749 msec
2024-12-06T01:55:09,981 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733450109981, completionTime=-1
2024-12-06T01:55:09,981 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:55:09,981 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:55:09,983 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:55:09,983 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733450169983
2024-12-06T01:55:09,983 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733450229983
2024-12-06T01:55:09,983 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T01:55:09,984 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36107,1733450108963-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,984 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36107,1733450108963-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,984 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36107,1733450108963-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,984 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:36107, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,984 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,985 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:09,986 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.929sec
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:55:09,991 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36107,1733450108963-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:55:09,992 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36107,1733450108963-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:55:09,995 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:55:09,995 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:55:09,995 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36107,1733450108963-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:55:10,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fc1909, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:55:10,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,36107,-1 for getting cluster id
2024-12-06T01:55:10,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:55:10,055 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1885f12a-8a09-4f80-aa0d-1cb6ac4c02b9'
2024-12-06T01:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1885f12a-8a09-4f80-aa0d-1cb6ac4c02b9"

2024-12-06T01:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd08f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,36107,-1]
2024-12-06T01:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:55:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:55:10,058 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:55:10,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df502bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:55:10,059 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:55:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45189,1733450109023, seqNum=-1]
2024-12-06T01:55:10,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:55:10,061 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:55:10,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:10,063 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:55:10,066 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:55:10,066 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-06T01:55:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,36107,1733450108963
2024-12-06T01:55:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54cc079f
2024-12-06T01:55:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-06T01:55:10,068 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-06T01:55:10,069 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-06T01:55:10,069 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-06T01:55:10,069 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:55:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-06T01:55:10,071 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-06T01:55:10,072 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:10,072 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-06T01:55:10,073 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-06T01:55:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:55:10,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741835_1011 (size=381)
2024-12-06T01:55:10,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741835_1011 (size=381)
2024-12-06T01:55:10,081 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff395bc6a25b16a53c8c3b4809bbcec1, NAME => 'TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db
2024-12-06T01:55:10,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741836_1012 (size=64)
2024-12-06T01:55:10,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741836_1012 (size=64)
2024-12-06T01:55:10,087 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:10,087 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing ff395bc6a25b16a53c8c3b4809bbcec1, disabling compactions & flushes
2024-12-06T01:55:10,087 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,087 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,087 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1. after waiting 0 ms
2024-12-06T01:55:10,087 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,087 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,087 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff395bc6a25b16a53c8c3b4809bbcec1:
Waiting for close lock at 1733450110087Disabling compacts and flushes for region at 1733450110087Disabling writes for close at 1733450110087Writing region close event to WAL at 1733450110087Closed at 1733450110087
2024-12-06T01:55:10,089 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-06T01:55:10,090 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733450110089"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733450110089"}]},"ts":"1733450110089"}
2024-12-06T01:55:10,092 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-06T01:55:10,093 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-06T01:55:10,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733450110093"}]},"ts":"1733450110093"}
2024-12-06T01:55:10,095 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-06T01:55:10,096 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, ASSIGN}]
2024-12-06T01:55:10,097 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, ASSIGN
2024-12-06T01:55:10,098 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, ASSIGN; state=OFFLINE, location=00c764f7c3aa,45189,1733450109023; forceNewPlan=false, retain=false
2024-12-06T01:55:10,222 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:10,249 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff395bc6a25b16a53c8c3b4809bbcec1, regionState=OPENING, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:10,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, ASSIGN because future has completed
2024-12-06T01:55:10,252 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023}]
2024-12-06T01:55:10,257 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:10,409 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,409 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ff395bc6a25b16a53c8c3b4809bbcec1, NAME => 'TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:55:10,410 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,410 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:10,410 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,410 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,411 INFO  [StoreOpener-ff395bc6a25b16a53c8c3b4809bbcec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ff395bc6a25b16a53c8c3b4809bbcec1 
2024-12-06T01:55:10,413 INFO  [StoreOpener-ff395bc6a25b16a53c8c3b4809bbcec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff395bc6a25b16a53c8c3b4809bbcec1 columnFamilyName info
2024-12-06T01:55:10,413 DEBUG [StoreOpener-ff395bc6a25b16a53c8c3b4809bbcec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:10,413 INFO  [StoreOpener-ff395bc6a25b16a53c8c3b4809bbcec1-1 {}] regionserver.HStore(327): Store=ff395bc6a25b16a53c8c3b4809bbcec1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:55:10,413 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,414 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,415 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,415 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,415 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,417 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,419 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:55:10,420 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ff395bc6a25b16a53c8c3b4809bbcec1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728396, jitterRate=-0.07379716634750366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:55:10,420 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:10,421 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ff395bc6a25b16a53c8c3b4809bbcec1:
Running coprocessor pre-open hook at 1733450110410Writing region info on filesystem at 1733450110410Initializing all the Stores at 1733450110411 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450110411Cleaning up temporary data from old regions at 1733450110415 (+4 ms)Running coprocessor post-open hooks at 1733450110420 (+5 ms)Region opened successfully at 1733450110421 (+1 ms)
2024-12-06T01:55:10,422 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., pid=6, masterSystemTime=1733450110405
2024-12-06T01:55:10,424 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,424 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:10,425 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff395bc6a25b16a53c8c3b4809bbcec1, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:10,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023 because future has completed
2024-12-06T01:55:10,431 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-06T01:55:10,431 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023 in 177 msec
2024-12-06T01:55:10,434 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-06T01:55:10,434 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, ASSIGN in 335 msec
2024-12-06T01:55:10,435 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-06T01:55:10,435 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733450110435"}]},"ts":"1733450110435"}
2024-12-06T01:55:10,437 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-06T01:55:10,438 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-06T01:55:10,441 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 369 msec
2024-12-06T01:55:11,222 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:11,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:11,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,731 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,731 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,731 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:11,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,223 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:12,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:12,271 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:55:12,273 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,273 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,273 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,274 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,274 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,274 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,275 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,275 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,294 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,296 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,296 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,300 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,300 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:12,304 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:13,224 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:13,259 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:13,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-06T01:55:13,405 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-06T01:55:13,406 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-06T01:55:14,224 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:14,260 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:15,225 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:15,260 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:15,307 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T01:55:15,308 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-06T01:55:16,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:16,261 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:17,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:17,262 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:18,227 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:18,262 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:18,909 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:55:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,940 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,942 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,942 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,949 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:18,953 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:19,227 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:19,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-06T01:55:20,139 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-06T01:55:20,139 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-06T01:55:20,142 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-06T01:55:20,142 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2]
2024-12-06T01:55:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:20,160 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:55:20,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/2a181c56408b47c886d77b526e5ba864 is 1080, key is row0001/info:/1733450120147/Put/seqid=0
2024-12-06T01:55:20,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741837_1013 (size=12509)
2024-12-06T01:55:20,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741837_1013 (size=12509)
2024-12-06T01:55:20,190 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/2a181c56408b47c886d77b526e5ba864
2024-12-06T01:55:20,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/2a181c56408b47c886d77b526e5ba864 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/2a181c56408b47c886d77b526e5ba864
2024-12-06T01:55:20,204 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T01:55:20,206 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/2a181c56408b47c886d77b526e5ba864, entries=7, sequenceid=11, filesize=12.2 K
2024-12-06T01:55:20,207 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for ff395bc6a25b16a53c8c3b4809bbcec1 in 47ms, sequenceid=11, compaction requested=false
2024-12-06T01:55:20,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36986 deadline: 1733450130203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:20,228 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 because the exception is null or not the one we care about
2024-12-06T01:55:20,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:21,229 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:21,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:22,229 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:22,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:23,230 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:23,265 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:24,231 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:24,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:25,231 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:25,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:26,232 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:26,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:27,233 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:27,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:28,233 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:28,268 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:29,234 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:29,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:30,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:30,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:30,281 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-06T01:55:30,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f46e0db75f034c439d90f0b6a21ea6eb is 1080, key is row0008/info:/1733450120161/Put/seqid=0
2024-12-06T01:55:30,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741838_1014 (size=29761)
2024-12-06T01:55:30,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741838_1014 (size=29761)
2024-12-06T01:55:30,293 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f46e0db75f034c439d90f0b6a21ea6eb
2024-12-06T01:55:30,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f46e0db75f034c439d90f0b6a21ea6eb as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb
2024-12-06T01:55:30,306 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb, entries=23, sequenceid=37, filesize=29.1 K
2024-12-06T01:55:30,307 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for ff395bc6a25b16a53c8c3b4809bbcec1 in 26ms, sequenceid=37, compaction requested=false
2024-12-06T01:55:30,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:30,307 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-12-06T01:55:30,307 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb because midkey is the same as first or last row
2024-12-06T01:55:31,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:31,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:32,236 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:32,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:32,297 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:55:32,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/4a50cb1d906a482c8b7c3cf0d6852571 is 1080, key is row0031/info:/1733450130282/Put/seqid=0
2024-12-06T01:55:32,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741839_1015 (size=12509)
2024-12-06T01:55:32,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741839_1015 (size=12509)
2024-12-06T01:55:32,323 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/4a50cb1d906a482c8b7c3cf0d6852571
2024-12-06T01:55:32,337 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T01:55:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36986 deadline: 1733450142336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 because the exception is null or not the one we care about
2024-12-06T01:55:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/4a50cb1d906a482c8b7c3cf0d6852571 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/4a50cb1d906a482c8b7c3cf0d6852571
2024-12-06T01:55:32,346 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/4a50cb1d906a482c8b7c3cf0d6852571, entries=7, sequenceid=47, filesize=12.2 K
2024-12-06T01:55:32,347 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for ff395bc6a25b16a53c8c3b4809bbcec1 in 50ms, sequenceid=47, compaction requested=true
2024-12-06T01:55:32,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:32,347 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-12-06T01:55:32,347 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:32,347 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb because midkey is the same as first or last row
2024-12-06T01:55:32,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff395bc6a25b16a53c8c3b4809bbcec1:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:55:32,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:32,348 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:55:32,353 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:55:32,353 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): ff395bc6a25b16a53c8c3b4809bbcec1/info is initiating minor compaction (all files)
2024-12-06T01:55:32,353 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff395bc6a25b16a53c8c3b4809bbcec1/info in TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:32,353 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/2a181c56408b47c886d77b526e5ba864, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/4a50cb1d906a482c8b7c3cf0d6852571] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp, totalSize=53.5 K
2024-12-06T01:55:32,354 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a181c56408b47c886d77b526e5ba864, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733450120147
2024-12-06T01:55:32,354 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting f46e0db75f034c439d90f0b6a21ea6eb, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733450120161
2024-12-06T01:55:32,355 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a50cb1d906a482c8b7c3cf0d6852571, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733450130282
2024-12-06T01:55:32,369 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff395bc6a25b16a53c8c3b4809bbcec1#info#compaction#57 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:55:32,370 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/b8922ec4e5cc4f818ae60c6a67a95a86 is 1080, key is row0001/info:/1733450120147/Put/seqid=0
2024-12-06T01:55:32,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741840_1016 (size=44978)
2024-12-06T01:55:32,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741840_1016 (size=44978)
2024-12-06T01:55:32,383 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/b8922ec4e5cc4f818ae60c6a67a95a86 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86
2024-12-06T01:55:32,390 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff395bc6a25b16a53c8c3b4809bbcec1/info of ff395bc6a25b16a53c8c3b4809bbcec1 into b8922ec4e5cc4f818ae60c6a67a95a86(size=43.9 K), total size for store is 43.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:32,390 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., storeName=ff395bc6a25b16a53c8c3b4809bbcec1/info, priority=13, startTime=1733450132347; duration=0sec
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=43.9 K, sizeToCheck=16.0 K
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 because midkey is the same as first or last row
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=43.9 K, sizeToCheck=16.0 K
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:32,390 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 because midkey is the same as first or last row
2024-12-06T01:55:32,391 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=43.9 K, sizeToCheck=16.0 K
2024-12-06T01:55:32,391 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:32,391 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 because midkey is the same as first or last row
2024-12-06T01:55:32,391 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:32,391 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff395bc6a25b16a53c8c3b4809bbcec1:info
2024-12-06T01:55:33,236 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:33,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:34,237 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:34,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:35,238 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:35,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:36,239 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:36,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:37,239 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:37,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:38,240 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:38,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:38,941 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:55:39,240 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:39,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:40,241 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:40,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:41,242 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:41,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:42,242 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:42,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:42,440 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-06T01:55:42,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f42ec11fbaae48c998212ef4a33595b6 is 1080, key is row0038/info:/1733450132298/Put/seqid=0
2024-12-06T01:55:42,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741841_1017 (size=29761)
2024-12-06T01:55:42,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741841_1017 (size=29761)
2024-12-06T01:55:42,452 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f42ec11fbaae48c998212ef4a33595b6
2024-12-06T01:55:42,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f42ec11fbaae48c998212ef4a33595b6 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f42ec11fbaae48c998212ef4a33595b6
2024-12-06T01:55:42,465 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f42ec11fbaae48c998212ef4a33595b6, entries=23, sequenceid=74, filesize=29.1 K
2024-12-06T01:55:42,466 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for ff395bc6a25b16a53c8c3b4809bbcec1 in 26ms, sequenceid=74, compaction requested=false
2024-12-06T01:55:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.0 K, sizeToCheck=16.0 K
2024-12-06T01:55:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 because midkey is the same as first or last row
2024-12-06T01:55:43,243 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:43,278 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:44,244 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:44,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,456 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:55:44,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/571cbf230b0e494c87320733c3ff5376 is 1080, key is row0061/info:/1733450142442/Put/seqid=0
2024-12-06T01:55:44,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741842_1018 (size=12509)
2024-12-06T01:55:44,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741842_1018 (size=12509)
2024-12-06T01:55:44,469 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/571cbf230b0e494c87320733c3ff5376
2024-12-06T01:55:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/571cbf230b0e494c87320733c3ff5376 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/571cbf230b0e494c87320733c3ff5376
2024-12-06T01:55:44,482 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/571cbf230b0e494c87320733c3ff5376, entries=7, sequenceid=84, filesize=12.2 K
2024-12-06T01:55:44,483 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for ff395bc6a25b16a53c8c3b4809bbcec1 in 26ms, sequenceid=84, compaction requested=true
2024-12-06T01:55:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=85.2 K, sizeToCheck=16.0 K
2024-12-06T01:55:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 because midkey is the same as first or last row
2024-12-06T01:55:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff395bc6a25b16a53c8c3b4809bbcec1:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:55:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:44,483 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:55:44,483 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-06T01:55:44,485 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 87248 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:55:44,485 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): ff395bc6a25b16a53c8c3b4809bbcec1/info is initiating minor compaction (all files)
2024-12-06T01:55:44,485 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff395bc6a25b16a53c8c3b4809bbcec1/info in TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:44,485 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f42ec11fbaae48c998212ef4a33595b6, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/571cbf230b0e494c87320733c3ff5376] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp, totalSize=85.2 K
2024-12-06T01:55:44,485 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8922ec4e5cc4f818ae60c6a67a95a86, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733450120147
2024-12-06T01:55:44,486 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting f42ec11fbaae48c998212ef4a33595b6, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733450132298
2024-12-06T01:55:44,486 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 571cbf230b0e494c87320733c3ff5376, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733450142442
2024-12-06T01:55:44,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f2b53c659c4642e89adcd27243c5f23c is 1080, key is row0068/info:/1733450144457/Put/seqid=0
2024-12-06T01:55:44,505 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff395bc6a25b16a53c8c3b4809bbcec1#info#compaction#61 average throughput is 22.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:55:44,505 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/b87d1b833ab44829843bcc477bf661de is 1080, key is row0001/info:/1733450120147/Put/seqid=0
2024-12-06T01:55:44,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741843_1019 (size=20064)
2024-12-06T01:55:44,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741843_1019 (size=20064)
2024-12-06T01:55:44,511 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f2b53c659c4642e89adcd27243c5f23c
2024-12-06T01:55:44,515 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741844_1020 (size=77532)
2024-12-06T01:55:44,515 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741844_1020 (size=77532)
2024-12-06T01:55:44,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/f2b53c659c4642e89adcd27243c5f23c as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f2b53c659c4642e89adcd27243c5f23c
2024-12-06T01:55:44,523 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/b87d1b833ab44829843bcc477bf661de as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de
2024-12-06T01:55:44,523 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f2b53c659c4642e89adcd27243c5f23c, entries=14, sequenceid=101, filesize=19.6 K
2024-12-06T01:55:44,525 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=15.76 KB/16140 for ff395bc6a25b16a53c8c3b4809bbcec1 in 41ms, sequenceid=101, compaction requested=false
2024-12-06T01:55:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.8 K, sizeToCheck=16.0 K
2024-12-06T01:55:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 because midkey is the same as first or last row
2024-12-06T01:55:44,530 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff395bc6a25b16a53c8c3b4809bbcec1/info of ff395bc6a25b16a53c8c3b4809bbcec1 into b87d1b833ab44829843bcc477bf661de(size=75.7 K), total size for store is 95.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:55:44,530 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff395bc6a25b16a53c8c3b4809bbcec1:

2024-12-06T01:55:44,530 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., storeName=ff395bc6a25b16a53c8c3b4809bbcec1/info, priority=13, startTime=1733450144483; duration=0sec
2024-12-06T01:55:44,530 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=95.3 K, sizeToCheck=16.0 K
2024-12-06T01:55:44,530 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:44,530 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=95.3 K, sizeToCheck=16.0 K
2024-12-06T01:55:44,530 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:44,531 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=95.3 K, sizeToCheck=16.0 K
2024-12-06T01:55:44,531 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-06T01:55:44,532 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:44,532 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:44,532 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff395bc6a25b16a53c8c3b4809bbcec1:info
2024-12-06T01:55:44,533 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] assignment.AssignmentManager(1363): Split request from 00c764f7c3aa,45189,1733450109023, parent={ENCODED => ff395bc6a25b16a53c8c3b4809bbcec1, NAME => 'TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-06T01:55:44,539 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:44,543 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ff395bc6a25b16a53c8c3b4809bbcec1, daughterA=6ce48c8d2f6a08259d933e8387571ad0, daughterB=f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:44,544 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ff395bc6a25b16a53c8c3b4809bbcec1, daughterA=6ce48c8d2f6a08259d933e8387571ad0, daughterB=f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:44,544 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ff395bc6a25b16a53c8c3b4809bbcec1, daughterA=6ce48c8d2f6a08259d933e8387571ad0, daughterB=f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:44,545 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ff395bc6a25b16a53c8c3b4809bbcec1, daughterA=6ce48c8d2f6a08259d933e8387571ad0, daughterB=f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:44,552 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, UNASSIGN}]
2024-12-06T01:55:44,553 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, UNASSIGN
2024-12-06T01:55:44,555 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff395bc6a25b16a53c8c3b4809bbcec1, regionState=CLOSING, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:44,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, UNASSIGN because future has completed
2024-12-06T01:55:44,558 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-06T01:55:44,558 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023}]
2024-12-06T01:55:44,716 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,716 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-06T01:55:44,717 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing ff395bc6a25b16a53c8c3b4809bbcec1, disabling compactions & flushes
2024-12-06T01:55:44,717 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:44,717 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:44,717 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1. after waiting 0 ms
2024-12-06T01:55:44,717 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:44,717 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing ff395bc6a25b16a53c8c3b4809bbcec1 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-06T01:55:44,722 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/bc60487fad9a46d6a01d00e26a17d2a0 is 1080, key is row0082/info:/1733450144484/Put/seqid=0
2024-12-06T01:55:44,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741845_1021 (size=21141)
2024-12-06T01:55:44,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741845_1021 (size=21141)
2024-12-06T01:55:44,730 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/bc60487fad9a46d6a01d00e26a17d2a0
2024-12-06T01:55:44,736 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/.tmp/info/bc60487fad9a46d6a01d00e26a17d2a0 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/bc60487fad9a46d6a01d00e26a17d2a0
2024-12-06T01:55:44,742 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/bc60487fad9a46d6a01d00e26a17d2a0, entries=15, sequenceid=120, filesize=20.6 K
2024-12-06T01:55:44,743 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=0 B/0 for ff395bc6a25b16a53c8c3b4809bbcec1 in 26ms, sequenceid=120, compaction requested=true
2024-12-06T01:55:44,745 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/2a181c56408b47c886d77b526e5ba864, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/4a50cb1d906a482c8b7c3cf0d6852571, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f42ec11fbaae48c998212ef4a33595b6, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/571cbf230b0e494c87320733c3ff5376] to archive
2024-12-06T01:55:44,746 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T01:55:44,748 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/2a181c56408b47c886d77b526e5ba864 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/2a181c56408b47c886d77b526e5ba864
2024-12-06T01:55:44,749 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f46e0db75f034c439d90f0b6a21ea6eb
2024-12-06T01:55:44,751 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b8922ec4e5cc4f818ae60c6a67a95a86
2024-12-06T01:55:44,752 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/4a50cb1d906a482c8b7c3cf0d6852571 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/4a50cb1d906a482c8b7c3cf0d6852571
2024-12-06T01:55:44,753 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f42ec11fbaae48c998212ef4a33595b6 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f42ec11fbaae48c998212ef4a33595b6
2024-12-06T01:55:44,755 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/571cbf230b0e494c87320733c3ff5376 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/571cbf230b0e494c87320733c3ff5376
2024-12-06T01:55:44,761 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/recovered.edits/123.seqid, newMaxSeqId=123, maxSeqId=1
2024-12-06T01:55:44,762 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.
2024-12-06T01:55:44,762 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for ff395bc6a25b16a53c8c3b4809bbcec1:
Waiting for close lock at 1733450144717Running coprocessor pre-close hooks at 1733450144717Disabling compacts and flushes for region at 1733450144717Disabling writes for close at 1733450144717Obtaining lock to block concurrent updates at 1733450144717Preparing flush snapshotting stores in ff395bc6a25b16a53c8c3b4809bbcec1 at 1733450144717Finished memstore snapshotting TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., syncing WAL and waiting on mvcc, flushsize=dataSize=16140, getHeapSize=17520, getOffHeapSize=0, getCellsCount=15 at 1733450144717Flushing stores of TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1. at 1733450144718 (+1 ms)Flushing ff395bc6a25b16a53c8c3b4809bbcec1/info: creating writer at 1733450144718Flushing ff395bc6a25b16a53c8c3b4809bbcec1/info: appending metadata at 1733450144721 (+3 ms)Flushing ff395bc6a25b16a53c8c3b4809bbcec1/info: closing flushed file at 1733450144721Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@795b3d78: reopening flushed file at 1733450144735 (+14 ms)Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=0 B/0 for ff395bc6a25b16a53c8c3b4809bbcec1 in 26ms, sequenceid=120, compaction requested=true at 1733450144743 (+8 ms)Writing region close event to WAL at 1733450144758 (+15 ms)Running coprocessor post-close hooks at 1733450144762 (+4 ms)Closed at 1733450144762
2024-12-06T01:55:44,765 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,766 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff395bc6a25b16a53c8c3b4809bbcec1, regionState=CLOSED
2024-12-06T01:55:44,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023 because future has completed
2024-12-06T01:55:44,771 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-06T01:55:44,772 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure ff395bc6a25b16a53c8c3b4809bbcec1, server=00c764f7c3aa,45189,1733450109023 in 211 msec
2024-12-06T01:55:44,774 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-06T01:55:44,774 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ff395bc6a25b16a53c8c3b4809bbcec1, UNASSIGN in 220 msec
2024-12-06T01:55:44,782 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:44,786 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=ff395bc6a25b16a53c8c3b4809bbcec1, threads=3
2024-12-06T01:55:44,789 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de for region: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,789 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/bc60487fad9a46d6a01d00e26a17d2a0 for region: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,789 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f2b53c659c4642e89adcd27243c5f23c for region: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,799 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/bc60487fad9a46d6a01d00e26a17d2a0, top=true
2024-12-06T01:55:44,799 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f2b53c659c4642e89adcd27243c5f23c, top=true
2024-12-06T01:55:44,807 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0 for child: f5ee4bc895492ff0809420da8a9aa7f5, parent: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741846_1022 (size=27)
2024-12-06T01:55:44,808 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/bc60487fad9a46d6a01d00e26a17d2a0 for region: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741846_1022 (size=27)
2024-12-06T01:55:44,809 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c for child: f5ee4bc895492ff0809420da8a9aa7f5, parent: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,809 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/f2b53c659c4642e89adcd27243c5f23c for region: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741847_1023 (size=27)
2024-12-06T01:55:44,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741847_1023 (size=27)
2024-12-06T01:55:44,820 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de for region: ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:55:44,823 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region ff395bc6a25b16a53c8c3b4809bbcec1 Daughter A: [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1] storefiles, Daughter B: [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c] storefiles.
2024-12-06T01:55:44,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741848_1024 (size=71)
2024-12-06T01:55:44,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741848_1024 (size=71)
2024-12-06T01:55:44,834 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:44,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741849_1025 (size=71)
2024-12-06T01:55:44,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741849_1025 (size=71)
2024-12-06T01:55:44,850 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:44,861 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/recovered.edits/123.seqid, newMaxSeqId=123, maxSeqId=-1
2024-12-06T01:55:44,863 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/recovered.edits/123.seqid, newMaxSeqId=123, maxSeqId=-1
2024-12-06T01:55:44,866 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733450144866"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733450144866"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733450144866"}]},"ts":"1733450144866"}
2024-12-06T01:55:44,866 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733450144866"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733450144866"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733450144866"}]},"ts":"1733450144866"}
2024-12-06T01:55:44,867 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733450144866"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733450144866"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733450144866"}]},"ts":"1733450144866"}
2024-12-06T01:55:44,886 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6ce48c8d2f6a08259d933e8387571ad0, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f5ee4bc895492ff0809420da8a9aa7f5, ASSIGN}]
2024-12-06T01:55:44,887 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6ce48c8d2f6a08259d933e8387571ad0, ASSIGN
2024-12-06T01:55:44,888 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f5ee4bc895492ff0809420da8a9aa7f5, ASSIGN
2024-12-06T01:55:44,888 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6ce48c8d2f6a08259d933e8387571ad0, ASSIGN; state=SPLITTING_NEW, location=00c764f7c3aa,45189,1733450109023; forceNewPlan=false, retain=false
2024-12-06T01:55:44,889 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f5ee4bc895492ff0809420da8a9aa7f5, ASSIGN; state=SPLITTING_NEW, location=00c764f7c3aa,45189,1733450109023; forceNewPlan=false, retain=false
2024-12-06T01:55:45,039 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f5ee4bc895492ff0809420da8a9aa7f5, regionState=OPENING, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:45,039 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ce48c8d2f6a08259d933e8387571ad0, regionState=OPENING, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:45,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6ce48c8d2f6a08259d933e8387571ad0, ASSIGN because future has completed
2024-12-06T01:55:45,043 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ce48c8d2f6a08259d933e8387571ad0, server=00c764f7c3aa,45189,1733450109023}]
2024-12-06T01:55:45,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f5ee4bc895492ff0809420da8a9aa7f5, ASSIGN because future has completed
2024-12-06T01:55:45,044 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023}]
2024-12-06T01:55:45,199 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:55:45,199 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => f5ee4bc895492ff0809420da8a9aa7f5, NAME => 'TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-06T01:55:45,200 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,200 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:45,200 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,200 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,201 INFO  [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f5ee4bc895492ff0809420da8a9aa7f5 
2024-12-06T01:55:45,202 INFO  [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5ee4bc895492ff0809420da8a9aa7f5 columnFamilyName info
2024-12-06T01:55:45,202 DEBUG [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:45,212 DEBUG [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0
2024-12-06T01:55:45,222 DEBUG [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c
2024-12-06T01:55:45,230 DEBUG [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1->hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de-top
2024-12-06T01:55:45,230 INFO  [StoreOpener-f5ee4bc895492ff0809420da8a9aa7f5-1 {}] regionserver.HStore(327): Store=f5ee4bc895492ff0809420da8a9aa7f5/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:55:45,231 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,232 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,233 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,234 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,234 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,236 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,237 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened f5ee4bc895492ff0809420da8a9aa7f5; next sequenceid=124; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702449, jitterRate=-0.10679106414318085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:55:45,237 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:45,237 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for f5ee4bc895492ff0809420da8a9aa7f5:
Running coprocessor pre-open hook at 1733450145200Writing region info on filesystem at 1733450145200Initializing all the Stores at 1733450145201 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450145201Cleaning up temporary data from old regions at 1733450145234 (+33 ms)Running coprocessor post-open hooks at 1733450145237 (+3 ms)Region opened successfully at 1733450145237
2024-12-06T01:55:45,238 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., pid=13, masterSystemTime=1733450145195
2024-12-06T01:55:45,239 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:55:45,239 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:55:45,239 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:45,240 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:55:45,240 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:55:45,240 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:55:45,241 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1->hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de-top, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=116.0 K
2024-12-06T01:55:45,241 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:55:45,241 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733450120147
2024-12-06T01:55:45,241 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:55:45,242 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:55:45,242 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ce48c8d2f6a08259d933e8387571ad0, NAME => 'TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-06T01:55:45,242 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733450144457
2024-12-06T01:55:45,242 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,242 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:55:45,242 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f5ee4bc895492ff0809420da8a9aa7f5, regionState=OPEN, openSeqNum=124, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:45,242 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,242 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,244 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733450144484
2024-12-06T01:55:45,245 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-06T01:55:45,245 INFO  [StoreOpener-6ce48c8d2f6a08259d933e8387571ad0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6ce48c8d2f6a08259d933e8387571ad0 
2024-12-06T01:55:45,244 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:45,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-06T01:55:45,245 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-12-06T01:55:45,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023 because future has completed
2024-12-06T01:55:45,246 INFO  [StoreOpener-6ce48c8d2f6a08259d933e8387571ad0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ce48c8d2f6a08259d933e8387571ad0 columnFamilyName info
2024-12-06T01:55:45,246 DEBUG [StoreOpener-6ce48c8d2f6a08259d933e8387571ad0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:55:45,250 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-12-06T01:55:45,250 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023 in 202 msec
2024-12-06T01:55:45,253 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f5ee4bc895492ff0809420da8a9aa7f5, ASSIGN in 364 msec
2024-12-06T01:55:45,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/info/a38b5bd432d24614b0db5c7b20ce17df is 193, key is TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5./info:regioninfo/1733450145242/Put/seqid=0
2024-12-06T01:55:45,279 DEBUG [StoreOpener-6ce48c8d2f6a08259d933e8387571ad0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1->hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de-bottom
2024-12-06T01:55:45,279 INFO  [StoreOpener-6ce48c8d2f6a08259d933e8387571ad0-1 {}] regionserver.HStore(327): Store=6ce48c8d2f6a08259d933e8387571ad0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:55:45,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:45,280 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,281 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,281 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#64 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:55:45,282 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/15406b748b4a4bd2b5648bf3e14db250 is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:55:45,282 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,283 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,283 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,285 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,287 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 6ce48c8d2f6a08259d933e8387571ad0; next sequenceid=124; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695601, jitterRate=-0.1154981255531311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-06T01:55:45,287 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:55:45,287 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6ce48c8d2f6a08259d933e8387571ad0:
Running coprocessor pre-open hook at 1733450145243Writing region info on filesystem at 1733450145243Initializing all the Stores at 1733450145244 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450145244Cleaning up temporary data from old regions at 1733450145283 (+39 ms)Running coprocessor post-open hooks at 1733450145287 (+4 ms)Region opened successfully at 1733450145287
2024-12-06T01:55:45,288 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0., pid=12, masterSystemTime=1733450145195
2024-12-06T01:55:45,288 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 6ce48c8d2f6a08259d933e8387571ad0:info, priority=-2147483648, current under compaction store size is 2
2024-12-06T01:55:45,288 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:45,288 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-06T01:55:45,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741851_1027 (size=42984)
2024-12-06T01:55:45,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741851_1027 (size=42984)
2024-12-06T01:55:45,290 INFO  [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:55:45,290 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HStore(1541): 6ce48c8d2f6a08259d933e8387571ad0/info is initiating minor compaction (all files)
2024-12-06T01:55:45,290 INFO  [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ce48c8d2f6a08259d933e8387571ad0/info in TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:55:45,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741850_1026 (size=9882)
2024-12-06T01:55:45,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741850_1026 (size=9882)
2024-12-06T01:55:45,290 INFO  [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1->hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de-bottom] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/.tmp, totalSize=75.7 K
2024-12-06T01:55:45,291 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:55:45,291 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] compactions.Compactor(225): Compacting b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733450120147
2024-12-06T01:55:45,291 INFO  [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:55:45,292 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/info/a38b5bd432d24614b0db5c7b20ce17df
2024-12-06T01:55:45,292 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ce48c8d2f6a08259d933e8387571ad0, regionState=OPEN, openSeqNum=124, regionLocation=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:45,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ce48c8d2f6a08259d933e8387571ad0, server=00c764f7c3aa,45189,1733450109023 because future has completed
2024-12-06T01:55:45,298 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/15406b748b4a4bd2b5648bf3e14db250 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/15406b748b4a4bd2b5648bf3e14db250
2024-12-06T01:55:45,301 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-12-06T01:55:45,301 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ce48c8d2f6a08259d933e8387571ad0, server=00c764f7c3aa,45189,1733450109023 in 255 msec
2024-12-06T01:55:45,304 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-12-06T01:55:45,304 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6ce48c8d2f6a08259d933e8387571ad0, ASSIGN in 415 msec
2024-12-06T01:55:45,307 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ff395bc6a25b16a53c8c3b4809bbcec1, daughterA=6ce48c8d2f6a08259d933e8387571ad0, daughterB=f5ee4bc895492ff0809420da8a9aa7f5 in 765 msec
2024-12-06T01:55:45,307 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into 15406b748b4a4bd2b5648bf3e14db250(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:55:45,307 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:55:45,307 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450145239; duration=0sec
2024-12-06T01:55:45,307 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:45,307 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:55:45,317 INFO  [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ce48c8d2f6a08259d933e8387571ad0#info#compaction#65 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:55:45,318 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/.tmp/info/95307e9c2f464e9e9be0d592ee9d3c4c is 1080, key is row0001/info:/1733450120147/Put/seqid=0
2024-12-06T01:55:45,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741852_1028 (size=70862)
2024-12-06T01:55:45,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741852_1028 (size=70862)
2024-12-06T01:55:45,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/ns/e14a764f7a1a4a468b249a40e0125b95 is 43, key is default/ns:d/1733450109975/Put/seqid=0
2024-12-06T01:55:45,338 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/.tmp/info/95307e9c2f464e9e9be0d592ee9d3c4c as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/95307e9c2f464e9e9be0d592ee9d3c4c
2024-12-06T01:55:45,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741853_1029 (size=5153)
2024-12-06T01:55:45,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741853_1029 (size=5153)
2024-12-06T01:55:45,340 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/ns/e14a764f7a1a4a468b249a40e0125b95
2024-12-06T01:55:45,345 INFO  [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 6ce48c8d2f6a08259d933e8387571ad0/info of 6ce48c8d2f6a08259d933e8387571ad0 into 95307e9c2f464e9e9be0d592ee9d3c4c(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:55:45,345 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ce48c8d2f6a08259d933e8387571ad0:

2024-12-06T01:55:45,345 INFO  [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0., storeName=6ce48c8d2f6a08259d933e8387571ad0/info, priority=15, startTime=1733450145288; duration=0sec
2024-12-06T01:55:45,345 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:45,345 DEBUG [RS:0;00c764f7c3aa:45189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ce48c8d2f6a08259d933e8387571ad0:info
2024-12-06T01:55:45,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/table/f569b6520caa435682f9b583c793316a is 65, key is TestLogRolling-testLogRolling/table:state/1733450110435/Put/seqid=0
2024-12-06T01:55:45,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741854_1030 (size=5340)
2024-12-06T01:55:45,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741854_1030 (size=5340)
2024-12-06T01:55:45,368 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/table/f569b6520caa435682f9b583c793316a
2024-12-06T01:55:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/info/a38b5bd432d24614b0db5c7b20ce17df as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/info/a38b5bd432d24614b0db5c7b20ce17df
2024-12-06T01:55:45,380 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/info/a38b5bd432d24614b0db5c7b20ce17df, entries=30, sequenceid=17, filesize=9.7 K
2024-12-06T01:55:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/ns/e14a764f7a1a4a468b249a40e0125b95 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/ns/e14a764f7a1a4a468b249a40e0125b95
2024-12-06T01:55:45,386 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/ns/e14a764f7a1a4a468b249a40e0125b95, entries=2, sequenceid=17, filesize=5.0 K
2024-12-06T01:55:45,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/table/f569b6520caa435682f9b583c793316a as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/table/f569b6520caa435682f9b583c793316a
2024-12-06T01:55:45,393 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/table/f569b6520caa435682f9b583c793316a, entries=2, sequenceid=17, filesize=5.2 K
2024-12-06T01:55:45,394 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 149ms, sequenceid=17, compaction requested=false
2024-12-06T01:55:45,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T01:55:46,245 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:46,280 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36986 deadline: 1733450156513, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1. is not online on 00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1. is not online on 00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1. is not online on 00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733450110069.ff395bc6a25b16a53c8c3b4809bbcec1., hostname=00c764f7c3aa,45189,1733450109023, seqNum=2 from cache
2024-12-06T01:55:47,246 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:47,280 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:48,246 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:48,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:49,247 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:49,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:49,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,766 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:49,794 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,247 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:50,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:50,303 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:55:50,305 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,305 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,307 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,307 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,328 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:55:50,637 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-06T01:55:50,637 INFO  [master/00c764f7c3aa:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-06T01:55:51,248 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:51,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:52,249 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:52,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:53,249 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:53,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:54,250 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:54,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:54,928 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-12-06T01:55:55,251 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:55,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:56,251 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:56,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124]
2024-12-06T01:55:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:56,641 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:55:56,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/939c462bc62b45dc9286d1a5470dd44d is 1080, key is row0097/info:/1733450156628/Put/seqid=0
2024-12-06T01:55:56,653 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741855_1031 (size=12515)
2024-12-06T01:55:56,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741855_1031 (size=12515)
2024-12-06T01:55:56,654 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/939c462bc62b45dc9286d1a5470dd44d
2024-12-06T01:55:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/939c462bc62b45dc9286d1a5470dd44d as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/939c462bc62b45dc9286d1a5470dd44d
2024-12-06T01:55:56,668 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/939c462bc62b45dc9286d1a5470dd44d, entries=7, sequenceid=134, filesize=12.2 K
2024-12-06T01:55:56,670 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for f5ee4bc895492ff0809420da8a9aa7f5 in 29ms, sequenceid=134, compaction requested=false
2024-12-06T01:55:56,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:55:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:56,671 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-06T01:55:56,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/88bfd42fb28447048aaa700b341a8acd is 1080, key is row0104/info:/1733450156642/Put/seqid=0
2024-12-06T01:55:56,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741856_1032 (size=15750)
2024-12-06T01:55:56,684 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/88bfd42fb28447048aaa700b341a8acd
2024-12-06T01:55:56,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741856_1032 (size=15750)
2024-12-06T01:55:56,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/88bfd42fb28447048aaa700b341a8acd as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/88bfd42fb28447048aaa700b341a8acd
2024-12-06T01:55:56,706 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/88bfd42fb28447048aaa700b341a8acd, entries=10, sequenceid=147, filesize=15.4 K
2024-12-06T01:55:56,706 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=15.76 KB/16140 for f5ee4bc895492ff0809420da8a9aa7f5 in 35ms, sequenceid=147, compaction requested=true
2024-12-06T01:55:56,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:55:56,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:55:56,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:56,707 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:55:56,708 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 71249 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:55:56,708 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:55:56,708 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:55:56,708 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/15406b748b4a4bd2b5648bf3e14db250, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/939c462bc62b45dc9286d1a5470dd44d, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/88bfd42fb28447048aaa700b341a8acd] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=69.6 K
2024-12-06T01:55:56,709 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15406b748b4a4bd2b5648bf3e14db250, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733450142444
2024-12-06T01:55:56,709 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939c462bc62b45dc9286d1a5470dd44d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733450156628
2024-12-06T01:55:56,709 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88bfd42fb28447048aaa700b341a8acd, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733450156642
2024-12-06T01:55:56,721 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#70 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:55:56,721 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/0b70db5718e340858466caa511e7b598 is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:55:56,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741857_1033 (size=61415)
2024-12-06T01:55:56,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741857_1033 (size=61415)
2024-12-06T01:55:56,734 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/0b70db5718e340858466caa511e7b598 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/0b70db5718e340858466caa511e7b598
2024-12-06T01:55:56,739 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into 0b70db5718e340858466caa511e7b598(size=60.0 K), total size for store is 60.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:55:56,739 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:55:56,739 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450156707; duration=0sec
2024-12-06T01:55:56,740 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:55:56,740 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:55:57,252 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:57,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:58,252 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:58,287 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:55:58,702 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-12-06T01:55:58,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/df685ed8c0d444468c51b71a3fe52422 is 1080, key is row0114/info:/1733450156672/Put/seqid=0
2024-12-06T01:55:58,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741858_1034 (size=22238)
2024-12-06T01:55:58,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741858_1034 (size=22238)
2024-12-06T01:55:58,714 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/df685ed8c0d444468c51b71a3fe52422
2024-12-06T01:55:58,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/df685ed8c0d444468c51b71a3fe52422 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/df685ed8c0d444468c51b71a3fe52422
2024-12-06T01:55:58,727 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T01:55:58,727 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/df685ed8c0d444468c51b71a3fe52422, entries=16, sequenceid=167, filesize=21.7 K
2024-12-06T01:55:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36986 deadline: 1733450168727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
2024-12-06T01:55:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 , the old value is region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:55:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 because the exception is null or not the one we care about
2024-12-06T01:55:58,728 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for f5ee4bc895492ff0809420da8a9aa7f5 in 26ms, sequenceid=167, compaction requested=false
2024-12-06T01:55:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:55:59,253 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:55:59,287 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:00,253 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:00,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:01,254 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:01,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:02,255 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:02,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:03,255 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:03,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:04,256 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:04,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:05,256 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:05,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:06,257 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:06,291 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:07,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:07,292 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:08,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:08,292 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:08,781 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-06T01:56:08,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/baac888fa8a74956b78966a2d4f166dd is 1080, key is row0130/info:/1733450158703/Put/seqid=0
2024-12-06T01:56:08,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741859_1035 (size=20078)
2024-12-06T01:56:08,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741859_1035 (size=20078)
2024-12-06T01:56:08,803 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/baac888fa8a74956b78966a2d4f166dd
2024-12-06T01:56:08,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/baac888fa8a74956b78966a2d4f166dd as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/baac888fa8a74956b78966a2d4f166dd
2024-12-06T01:56:08,814 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T01:56:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36986 deadline: 1733450178814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
2024-12-06T01:56:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 , the old value is region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:56:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:56:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 because the exception is null or not the one we care about
2024-12-06T01:56:08,821 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/baac888fa8a74956b78966a2d4f166dd, entries=14, sequenceid=184, filesize=19.6 K
2024-12-06T01:56:08,822 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=15.76 KB/16140 for f5ee4bc895492ff0809420da8a9aa7f5 in 41ms, sequenceid=184, compaction requested=true
2024-12-06T01:56:08,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:08,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:56:08,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:08,823 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:56:08,824 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 103731 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:56:08,824 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:56:08,824 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:08,824 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/0b70db5718e340858466caa511e7b598, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/df685ed8c0d444468c51b71a3fe52422, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/baac888fa8a74956b78966a2d4f166dd] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=101.3 K
2024-12-06T01:56:08,825 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b70db5718e340858466caa511e7b598, keycount=52, bloomtype=ROW, size=60.0 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733450142444
2024-12-06T01:56:08,825 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting df685ed8c0d444468c51b71a3fe52422, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733450156672
2024-12-06T01:56:08,826 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting baac888fa8a74956b78966a2d4f166dd, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733450158703
2024-12-06T01:56:08,842 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#73 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:56:08,843 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/ae528524c2af4115b7c9c49ccbe3cf3c is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:56:08,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741860_1036 (size=93998)
2024-12-06T01:56:08,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741860_1036 (size=93998)
2024-12-06T01:56:08,857 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/ae528524c2af4115b7c9c49ccbe3cf3c as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/ae528524c2af4115b7c9c49ccbe3cf3c
2024-12-06T01:56:08,864 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into ae528524c2af4115b7c9c49ccbe3cf3c(size=91.8 K), total size for store is 91.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:56:08,864 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:08,864 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450168823; duration=0sec
2024-12-06T01:56:08,864 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:08,864 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:56:08,942 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-06T01:56:09,259 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:09,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:10,167 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache
2024-12-06T01:56:10,259 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:10,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:11,260 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:11,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:12,261 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:12,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:13,261 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:13,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:14,024 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=9, reused chunk count=61, reuseRatio=87.14%
2024-12-06T01:56:14,025 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-06T01:56:14,262 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:14,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:15,262 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:15,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:16,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:16,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:17,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:17,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:18,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:18,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:18,881 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-12-06T01:56:18,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/5bd16428f88a479fadcd84694d0ad032 is 1080, key is row0144/info:/1733450168782/Put/seqid=0
2024-12-06T01:56:18,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741861_1037 (size=22238)
2024-12-06T01:56:18,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741861_1037 (size=22238)
2024-12-06T01:56:18,892 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/5bd16428f88a479fadcd84694d0ad032
2024-12-06T01:56:18,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/5bd16428f88a479fadcd84694d0ad032 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/5bd16428f88a479fadcd84694d0ad032
2024-12-06T01:56:18,903 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/5bd16428f88a479fadcd84694d0ad032, entries=16, sequenceid=204, filesize=21.7 K
2024-12-06T01:56:18,904 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=1.05 KB/1076 for f5ee4bc895492ff0809420da8a9aa7f5 in 24ms, sequenceid=204, compaction requested=false
2024-12-06T01:56:18,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:19,265 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:19,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:20,265 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:20,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:20,895 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:56:20,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/8bb4a5a34d024693912f5d82213c587f is 1080, key is row0160/info:/1733450178882/Put/seqid=0
2024-12-06T01:56:20,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741862_1038 (size=12516)
2024-12-06T01:56:20,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741862_1038 (size=12516)
2024-12-06T01:56:20,906 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/8bb4a5a34d024693912f5d82213c587f
2024-12-06T01:56:20,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/8bb4a5a34d024693912f5d82213c587f as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/8bb4a5a34d024693912f5d82213c587f
2024-12-06T01:56:20,930 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/8bb4a5a34d024693912f5d82213c587f, entries=7, sequenceid=214, filesize=12.2 K
2024-12-06T01:56:20,931 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for f5ee4bc895492ff0809420da8a9aa7f5 in 36ms, sequenceid=214, compaction requested=true
2024-12-06T01:56:20,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:20,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:56:20,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:20,932 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:56:20,933 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128752 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:56:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:20,933 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:56:20,933 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB
2024-12-06T01:56:20,933 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:20,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,933 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/ae528524c2af4115b7c9c49ccbe3cf3c, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/5bd16428f88a479fadcd84694d0ad032, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/8bb4a5a34d024693912f5d82213c587f] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=125.7 K
2024-12-06T01:56:20,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,934 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae528524c2af4115b7c9c49ccbe3cf3c, keycount=82, bloomtype=ROW, size=91.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733450142444
2024-12-06T01:56:20,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,934 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd16428f88a479fadcd84694d0ad032, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733450168782
2024-12-06T01:56:20,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,935 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb4a5a34d024693912f5d82213c587f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733450178882
2024-12-06T01:56:20,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/6902fb53d30a49d1bbf232061006ba64 is 1080, key is row0167/info:/1733450180896/Put/seqid=0
2024-12-06T01:56:20,939 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,939 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,944 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741863_1039 (size=27628)
2024-12-06T01:56:20,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741863_1039 (size=27628)
2024-12-06T01:56:20,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:20,947 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/6902fb53d30a49d1bbf232061006ba64
2024-12-06T01:56:20,951 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#77 average throughput is 53.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:56:20,952 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/09b33acf0e6440c48deb3a70dab6cd74 is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:56:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/6902fb53d30a49d1bbf232061006ba64 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6902fb53d30a49d1bbf232061006ba64
2024-12-06T01:56:20,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741864_1040 (size=118898)
2024-12-06T01:56:20,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741864_1040 (size=118898)
2024-12-06T01:56:20,964 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6902fb53d30a49d1bbf232061006ba64, entries=21, sequenceid=238, filesize=27.0 K
2024-12-06T01:56:20,965 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=5.25 KB/5380 for f5ee4bc895492ff0809420da8a9aa7f5 in 32ms, sequenceid=238, compaction requested=false
2024-12-06T01:56:20,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:20,968 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/09b33acf0e6440c48deb3a70dab6cd74 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/09b33acf0e6440c48deb3a70dab6cd74
2024-12-06T01:56:20,974 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into 09b33acf0e6440c48deb3a70dab6cd74(size=116.1 K), total size for store is 143.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:56:20,974 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:20,974 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450180931; duration=0sec
2024-12-06T01:56:20,974 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:20,974 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:56:21,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:21,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:21,456 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-06T01:56:21,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,490 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,491 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,491 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,492 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,492 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,492 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:21,503 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-06T01:56:22,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:22,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:22,949 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:56:22,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/983eabbc2e654fb8b0e62c0b43160225 is 1080, key is row0188/info:/1733450180934/Put/seqid=0
2024-12-06T01:56:22,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741865_1041 (size=12516)
2024-12-06T01:56:22,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741865_1041 (size=12516)
2024-12-06T01:56:22,971 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/983eabbc2e654fb8b0e62c0b43160225
2024-12-06T01:56:22,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/983eabbc2e654fb8b0e62c0b43160225 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/983eabbc2e654fb8b0e62c0b43160225
2024-12-06T01:56:22,982 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/983eabbc2e654fb8b0e62c0b43160225, entries=7, sequenceid=249, filesize=12.2 K
2024-12-06T01:56:22,983 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for f5ee4bc895492ff0809420da8a9aa7f5 in 34ms, sequenceid=249, compaction requested=true
2024-12-06T01:56:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:56:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:22,983 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:56:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:22,984 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-06T01:56:22,985 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 159042 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:56:22,985 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:56:22,985 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:22,985 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/09b33acf0e6440c48deb3a70dab6cd74, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6902fb53d30a49d1bbf232061006ba64, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/983eabbc2e654fb8b0e62c0b43160225] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=155.3 K
2024-12-06T01:56:22,986 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b33acf0e6440c48deb3a70dab6cd74, keycount=105, bloomtype=ROW, size=116.1 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733450142444
2024-12-06T01:56:22,986 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6902fb53d30a49d1bbf232061006ba64, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733450180896
2024-12-06T01:56:22,986 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 983eabbc2e654fb8b0e62c0b43160225, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733450180934
2024-12-06T01:56:22,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/a1e428364dcf4d0d92ffeaa80458e8bc is 1080, key is row0195/info:/1733450182950/Put/seqid=0
2024-12-06T01:56:23,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741866_1042 (size=24407)
2024-12-06T01:56:23,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741866_1042 (size=24407)
2024-12-06T01:56:23,004 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/a1e428364dcf4d0d92ffeaa80458e8bc
2024-12-06T01:56:23,009 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#80 average throughput is 45.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:56:23,010 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-06T01:56:23,010 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/47864fbee1bc41c1933e066a70df7ca8 is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:56:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36986 deadline: 1733450193010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
2024-12-06T01:56:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 , the old value is region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:56:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f5ee4bc895492ff0809420da8a9aa7f5, server=00c764f7c3aa,45189,1733450109023
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-06T01:56:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., hostname=00c764f7c3aa,45189,1733450109023, seqNum=124 because the exception is null or not the one we care about
2024-12-06T01:56:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/a1e428364dcf4d0d92ffeaa80458e8bc as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/a1e428364dcf4d0d92ffeaa80458e8bc
2024-12-06T01:56:23,024 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/a1e428364dcf4d0d92ffeaa80458e8bc, entries=18, sequenceid=270, filesize=23.8 K
2024-12-06T01:56:23,025 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=11.56 KB/11836 for f5ee4bc895492ff0809420da8a9aa7f5 in 41ms, sequenceid=270, compaction requested=false
2024-12-06T01:56:23,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:23,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741867_1043 (size=149389)
2024-12-06T01:56:23,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741867_1043 (size=149389)
2024-12-06T01:56:23,049 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/47864fbee1bc41c1933e066a70df7ca8 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/47864fbee1bc41c1933e066a70df7ca8
2024-12-06T01:56:23,056 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into 47864fbee1bc41c1933e066a70df7ca8(size=145.9 K), total size for store is 169.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:56:23,056 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:23,056 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450182983; duration=0sec
2024-12-06T01:56:23,056 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:23,056 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:56:23,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:23,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:24,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:24,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:25,268 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:25,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:26,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:26,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:27,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:27,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:28,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:28,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:29,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:29,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:30,200 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region f5ee4bc895492ff0809420da8a9aa7f5, had cached 0 bytes from a total of 173796
2024-12-06T01:56:30,242 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6ce48c8d2f6a08259d933e8387571ad0, had cached 0 bytes from a total of 70862
2024-12-06T01:56:30,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:30,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:31,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:31,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:32,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:32,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:33,100 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-06T01:56:33,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/4bf6eda958554e138e277c0dd0ef337e is 1080, key is row0213/info:/1733450182985/Put/seqid=0
2024-12-06T01:56:33,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741868_1044 (size=17918)
2024-12-06T01:56:33,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741868_1044 (size=17918)
2024-12-06T01:56:33,112 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/4bf6eda958554e138e277c0dd0ef337e
2024-12-06T01:56:33,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/4bf6eda958554e138e277c0dd0ef337e as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/4bf6eda958554e138e277c0dd0ef337e
2024-12-06T01:56:33,124 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/4bf6eda958554e138e277c0dd0ef337e, entries=12, sequenceid=286, filesize=17.5 K
2024-12-06T01:56:33,125 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for f5ee4bc895492ff0809420da8a9aa7f5 in 25ms, sequenceid=286, compaction requested=true
2024-12-06T01:56:33,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:33,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:56:33,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:33,125 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:56:33,126 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 191714 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:56:33,126 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:56:33,127 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:33,127 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/47864fbee1bc41c1933e066a70df7ca8, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/a1e428364dcf4d0d92ffeaa80458e8bc, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/4bf6eda958554e138e277c0dd0ef337e] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=187.2 K
2024-12-06T01:56:33,127 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47864fbee1bc41c1933e066a70df7ca8, keycount=133, bloomtype=ROW, size=145.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733450142444
2024-12-06T01:56:33,127 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e428364dcf4d0d92ffeaa80458e8bc, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733450182950
2024-12-06T01:56:33,128 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf6eda958554e138e277c0dd0ef337e, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733450182985
2024-12-06T01:56:33,143 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#82 average throughput is 83.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:56:33,144 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/dce5f8d4ae264015b4e7585163eeeab2 is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:56:33,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741869_1045 (size=181864)
2024-12-06T01:56:33,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741869_1045 (size=181864)
2024-12-06T01:56:33,164 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/dce5f8d4ae264015b4e7585163eeeab2 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/dce5f8d4ae264015b4e7585163eeeab2
2024-12-06T01:56:33,171 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into dce5f8d4ae264015b4e7585163eeeab2(size=177.6 K), total size for store is 177.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:56:33,171 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:33,171 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450193125; duration=0sec
2024-12-06T01:56:33,171 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:33,171 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:56:33,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:33,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:34,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:34,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:35,112 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-06T01:56:35,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/6201527d39ab4632b21d922f516e2fc7 is 1080, key is row0225/info:/1733450195102/Put/seqid=0
2024-12-06T01:56:35,126 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741870_1046 (size=12523)
2024-12-06T01:56:35,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741870_1046 (size=12523)
2024-12-06T01:56:35,127 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/6201527d39ab4632b21d922f516e2fc7
2024-12-06T01:56:35,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/6201527d39ab4632b21d922f516e2fc7 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6201527d39ab4632b21d922f516e2fc7
2024-12-06T01:56:35,139 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6201527d39ab4632b21d922f516e2fc7, entries=7, sequenceid=297, filesize=12.2 K
2024-12-06T01:56:35,140 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for f5ee4bc895492ff0809420da8a9aa7f5 in 28ms, sequenceid=297, compaction requested=false
2024-12-06T01:56:35,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45189 {}] regionserver.HRegion(8855): Flush requested on f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:35,142 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-12-06T01:56:35,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/9724c9dc63db4dceb1047c9d57cb6038 is 1080, key is row0232/info:/1733450195113/Put/seqid=0
2024-12-06T01:56:35,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741871_1047 (size=22254)
2024-12-06T01:56:35,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741871_1047 (size=22254)
2024-12-06T01:56:35,153 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/9724c9dc63db4dceb1047c9d57cb6038
2024-12-06T01:56:35,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/9724c9dc63db4dceb1047c9d57cb6038 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/9724c9dc63db4dceb1047c9d57cb6038
2024-12-06T01:56:35,164 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/9724c9dc63db4dceb1047c9d57cb6038, entries=16, sequenceid=316, filesize=21.7 K
2024-12-06T01:56:35,165 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=9.46 KB/9684 for f5ee4bc895492ff0809420da8a9aa7f5 in 23ms, sequenceid=316, compaction requested=true
2024-12-06T01:56:35,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:35,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5ee4bc895492ff0809420da8a9aa7f5:info, priority=-2147483648, current under compaction store size is 1
2024-12-06T01:56:35,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:35,166 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-06T01:56:35,167 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 216641 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-06T01:56:35,167 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1541): f5ee4bc895492ff0809420da8a9aa7f5/info is initiating minor compaction (all files)
2024-12-06T01:56:35,167 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5ee4bc895492ff0809420da8a9aa7f5/info in TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:35,167 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/dce5f8d4ae264015b4e7585163eeeab2, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6201527d39ab4632b21d922f516e2fc7, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/9724c9dc63db4dceb1047c9d57cb6038] into tmpdir=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp, totalSize=211.6 K
2024-12-06T01:56:35,167 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting dce5f8d4ae264015b4e7585163eeeab2, keycount=163, bloomtype=ROW, size=177.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733450142444
2024-12-06T01:56:35,168 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6201527d39ab4632b21d922f516e2fc7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733450195102
2024-12-06T01:56:35,168 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9724c9dc63db4dceb1047c9d57cb6038, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733450195113
2024-12-06T01:56:35,182 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5ee4bc895492ff0809420da8a9aa7f5#info#compaction#85 average throughput is 38.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-06T01:56:35,184 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/41b5e3b39f7541bc8ffb958753771474 is 1080, key is row0062/info:/1733450142444/Put/seqid=0
2024-12-06T01:56:35,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741872_1048 (size=206860)
2024-12-06T01:56:35,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741872_1048 (size=206860)
2024-12-06T01:56:35,192 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/41b5e3b39f7541bc8ffb958753771474 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/41b5e3b39f7541bc8ffb958753771474
2024-12-06T01:56:35,198 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5ee4bc895492ff0809420da8a9aa7f5/info of f5ee4bc895492ff0809420da8a9aa7f5 into 41b5e3b39f7541bc8ffb958753771474(size=202.0 K), total size for store is 202.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-06T01:56:35,198 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:35,198 INFO  [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., storeName=f5ee4bc895492ff0809420da8a9aa7f5/info, priority=13, startTime=1733450195166; duration=0sec
2024-12-06T01:56:35,198 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-06T01:56:35,198 DEBUG [RS:0;00c764f7c3aa:45189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5ee4bc895492ff0809420da8a9aa7f5:info
2024-12-06T01:56:35,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:35,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:36,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:36,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:37,159 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-06T01:56:37,159 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45189%2C1733450109023.1733450197159
2024-12-06T01:56:37,177 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,178 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,178 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,178 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,178 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,178 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450109474 with entries=308, filesize=307.10 KB; new WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450197159
2024-12-06T01:56:37,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33787:33787),(127.0.0.1/127.0.0.1:35651:35651)]
2024-12-06T01:56:37,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450109474 is not closed yet, will try archiving it next time
2024-12-06T01:56:37,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741833_1009 (size=314476)
2024-12-06T01:56:37,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741833_1009 (size=314476)
2024-12-06T01:56:37,183 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-12-06T01:56:37,189 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/info/3c4688efa382495686e6518fd5b11477 is 186, key is TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0./info:regioninfo/1733450145292/Put/seqid=0
2024-12-06T01:56:37,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741874_1050 (size=6153)
2024-12-06T01:56:37,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741874_1050 (size=6153)
2024-12-06T01:56:37,198 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/info/3c4688efa382495686e6518fd5b11477
2024-12-06T01:56:37,204 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/.tmp/info/3c4688efa382495686e6518fd5b11477 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/info/3c4688efa382495686e6518fd5b11477
2024-12-06T01:56:37,210 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/info/3c4688efa382495686e6518fd5b11477, entries=5, sequenceid=21, filesize=6.0 K
2024-12-06T01:56:37,211 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=21, compaction requested=false
2024-12-06T01:56:37,211 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-06T01:56:37,211 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing f5ee4bc895492ff0809420da8a9aa7f5 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-06T01:56:37,215 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/c7db503881f64475a65feb792e55b81e is 1080, key is row0248/info:/1733450195143/Put/seqid=0
2024-12-06T01:56:37,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741875_1051 (size=14681)
2024-12-06T01:56:37,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741875_1051 (size=14681)
2024-12-06T01:56:37,221 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/c7db503881f64475a65feb792e55b81e
2024-12-06T01:56:37,226 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/.tmp/info/c7db503881f64475a65feb792e55b81e as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/c7db503881f64475a65feb792e55b81e
2024-12-06T01:56:37,232 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/c7db503881f64475a65feb792e55b81e, entries=9, sequenceid=329, filesize=14.3 K
2024-12-06T01:56:37,233 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for f5ee4bc895492ff0809420da8a9aa7f5 in 22ms, sequenceid=329, compaction requested=false
2024-12-06T01:56:37,233 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f5ee4bc895492ff0809420da8a9aa7f5:

2024-12-06T01:56:37,233 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6ce48c8d2f6a08259d933e8387571ad0:

2024-12-06T01:56:37,233 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C45189%2C1733450109023.1733450197233
2024-12-06T01:56:37,238 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,238 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,238 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,238 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,238 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,238 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450197159 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450197233
2024-12-06T01:56:37,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35651:35651),(127.0.0.1/127.0.0.1:33787:33787)]
2024-12-06T01:56:37,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741873_1049 (size=731)
2024-12-06T01:56:37,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450197159 is not closed yet, will try archiving it next time
2024-12-06T01:56:37,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741873_1049 (size=731)
2024-12-06T01:56:37,240 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450109474 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/oldWALs/00c764f7c3aa%2C45189%2C1733450109023.1733450109474
2024-12-06T01:56:37,241 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-06T01:56:37,242 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/WALs/00c764f7c3aa,45189,1733450109023/00c764f7c3aa%2C45189%2C1733450109023.1733450197159 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/oldWALs/00c764f7c3aa%2C45189%2C1733450109023.1733450197159
2024-12-06T01:56:37,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:37,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:37,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:56:37,342 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:56:37,342 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:56:37,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:37,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:37,342 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:56:37,342 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:56:37,342 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1908543013, stopped=false
2024-12-06T01:56:37,343 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,36107,1733450108963
2024-12-06T01:56:37,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:56:37,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:56:37,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:37,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:37,344 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:56:37,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:56:37,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:56:37,345 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:56:37,345 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:56:37,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:37,345 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,45189,1733450109023' *****
2024-12-06T01:56:37,346 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:56:37,346 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(3091): Received CLOSE for f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ce48c8d2f6a08259d933e8387571ad0
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,45189,1733450109023
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:45189.
2024-12-06T01:56:37,346 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5ee4bc895492ff0809420da8a9aa7f5, disabling compactions & flushes
2024-12-06T01:56:37,346 DEBUG [RS:0;00c764f7c3aa:45189 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:56:37,346 DEBUG [RS:0;00c764f7c3aa:45189 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:37,346 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:37,346 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:37,346 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5. after waiting 0 ms
2024-12-06T01:56:37,346 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:37,346 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:56:37,347 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:56:37,347 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:56:37,347 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:56:37,347 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-06T01:56:37,347 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f5ee4bc895492ff0809420da8a9aa7f5=TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5., 6ce48c8d2f6a08259d933e8387571ad0=TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.}
2024-12-06T01:56:37,347 DEBUG [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6ce48c8d2f6a08259d933e8387571ad0, f5ee4bc895492ff0809420da8a9aa7f5
2024-12-06T01:56:37,347 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:56:37,347 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:56:37,347 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:56:37,347 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:56:37,347 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:56:37,347 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1->hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de-top, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/15406b748b4a4bd2b5648bf3e14db250, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/939c462bc62b45dc9286d1a5470dd44d, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/0b70db5718e340858466caa511e7b598, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/88bfd42fb28447048aaa700b341a8acd, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/df685ed8c0d444468c51b71a3fe52422, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/ae528524c2af4115b7c9c49ccbe3cf3c, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/baac888fa8a74956b78966a2d4f166dd, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/5bd16428f88a479fadcd84694d0ad032, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/09b33acf0e6440c48deb3a70dab6cd74, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/8bb4a5a34d024693912f5d82213c587f, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6902fb53d30a49d1bbf232061006ba64, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/47864fbee1bc41c1933e066a70df7ca8, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/983eabbc2e654fb8b0e62c0b43160225, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/a1e428364dcf4d0d92ffeaa80458e8bc, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/dce5f8d4ae264015b4e7585163eeeab2, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/4bf6eda958554e138e277c0dd0ef337e, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6201527d39ab4632b21d922f516e2fc7, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/9724c9dc63db4dceb1047c9d57cb6038] to archive
2024-12-06T01:56:37,349 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T01:56:37,351 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:56:37,352 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-f2b53c659c4642e89adcd27243c5f23c
2024-12-06T01:56:37,353 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/15406b748b4a4bd2b5648bf3e14db250 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/15406b748b4a4bd2b5648bf3e14db250
2024-12-06T01:56:37,355 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/TestLogRolling-testLogRolling=ff395bc6a25b16a53c8c3b4809bbcec1-bc60487fad9a46d6a01d00e26a17d2a0
2024-12-06T01:56:37,356 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/939c462bc62b45dc9286d1a5470dd44d to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/939c462bc62b45dc9286d1a5470dd44d
2024-12-06T01:56:37,358 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/0b70db5718e340858466caa511e7b598 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/0b70db5718e340858466caa511e7b598
2024-12-06T01:56:37,359 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/88bfd42fb28447048aaa700b341a8acd to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/88bfd42fb28447048aaa700b341a8acd
2024-12-06T01:56:37,361 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/df685ed8c0d444468c51b71a3fe52422 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/df685ed8c0d444468c51b71a3fe52422
2024-12-06T01:56:37,362 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/ae528524c2af4115b7c9c49ccbe3cf3c to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/ae528524c2af4115b7c9c49ccbe3cf3c
2024-12-06T01:56:37,364 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/baac888fa8a74956b78966a2d4f166dd to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/baac888fa8a74956b78966a2d4f166dd
2024-12-06T01:56:37,365 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-06T01:56:37,365 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:56:37,365 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:56:37,365 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/5bd16428f88a479fadcd84694d0ad032 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/5bd16428f88a479fadcd84694d0ad032
2024-12-06T01:56:37,365 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450197347Running coprocessor pre-close hooks at 1733450197347Disabling compacts and flushes for region at 1733450197347Disabling writes for close at 1733450197347Writing region close event to WAL at 1733450197360 (+13 ms)Running coprocessor post-close hooks at 1733450197365 (+5 ms)Closed at 1733450197365
2024-12-06T01:56:37,366 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:56:37,367 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/09b33acf0e6440c48deb3a70dab6cd74 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/09b33acf0e6440c48deb3a70dab6cd74
2024-12-06T01:56:37,368 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/8bb4a5a34d024693912f5d82213c587f to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/8bb4a5a34d024693912f5d82213c587f
2024-12-06T01:56:37,370 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6902fb53d30a49d1bbf232061006ba64 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6902fb53d30a49d1bbf232061006ba64
2024-12-06T01:56:37,371 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/47864fbee1bc41c1933e066a70df7ca8 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/47864fbee1bc41c1933e066a70df7ca8
2024-12-06T01:56:37,373 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/983eabbc2e654fb8b0e62c0b43160225 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/983eabbc2e654fb8b0e62c0b43160225
2024-12-06T01:56:37,374 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/a1e428364dcf4d0d92ffeaa80458e8bc to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/a1e428364dcf4d0d92ffeaa80458e8bc
2024-12-06T01:56:37,375 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/dce5f8d4ae264015b4e7585163eeeab2 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/dce5f8d4ae264015b4e7585163eeeab2
2024-12-06T01:56:37,376 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/4bf6eda958554e138e277c0dd0ef337e to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/4bf6eda958554e138e277c0dd0ef337e
2024-12-06T01:56:37,378 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6201527d39ab4632b21d922f516e2fc7 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/6201527d39ab4632b21d922f516e2fc7
2024-12-06T01:56:37,379 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/9724c9dc63db4dceb1047c9d57cb6038 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/info/9724c9dc63db4dceb1047c9d57cb6038
2024-12-06T01:56:37,379 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=00c764f7c3aa:36107 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-12-06T01:56:37,380 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [15406b748b4a4bd2b5648bf3e14db250=42984, 939c462bc62b45dc9286d1a5470dd44d=12515, 0b70db5718e340858466caa511e7b598=61415, 88bfd42fb28447048aaa700b341a8acd=15750, df685ed8c0d444468c51b71a3fe52422=22238, ae528524c2af4115b7c9c49ccbe3cf3c=93998, baac888fa8a74956b78966a2d4f166dd=20078, 5bd16428f88a479fadcd84694d0ad032=22238, 09b33acf0e6440c48deb3a70dab6cd74=118898, 8bb4a5a34d024693912f5d82213c587f=12516, 6902fb53d30a49d1bbf232061006ba64=27628, 47864fbee1bc41c1933e066a70df7ca8=149389, 983eabbc2e654fb8b0e62c0b43160225=12516, a1e428364dcf4d0d92ffeaa80458e8bc=24407, dce5f8d4ae264015b4e7585163eeeab2=181864, 4bf6eda958554e138e277c0dd0ef337e=17918, 6201527d39ab4632b21d922f516e2fc7=12523, 9724c9dc63db4dceb1047c9d57cb6038=22254]
2024-12-06T01:56:37,383 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/f5ee4bc895492ff0809420da8a9aa7f5/recovered.edits/332.seqid, newMaxSeqId=332, maxSeqId=123
2024-12-06T01:56:37,384 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:37,384 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5ee4bc895492ff0809420da8a9aa7f5:
Waiting for close lock at 1733450197346Running coprocessor pre-close hooks at 1733450197346Disabling compacts and flushes for region at 1733450197346Disabling writes for close at 1733450197346Writing region close event to WAL at 1733450197380 (+34 ms)Running coprocessor post-close hooks at 1733450197384 (+4 ms)Closed at 1733450197384
2024-12-06T01:56:37,384 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733450144539.f5ee4bc895492ff0809420da8a9aa7f5.
2024-12-06T01:56:37,384 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ce48c8d2f6a08259d933e8387571ad0, disabling compactions & flushes
2024-12-06T01:56:37,384 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:56:37,384 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:56:37,385 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0. after waiting 0 ms
2024-12-06T01:56:37,385 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:56:37,385 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1->hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/ff395bc6a25b16a53c8c3b4809bbcec1/info/b87d1b833ab44829843bcc477bf661de-bottom] to archive
2024-12-06T01:56:37,386 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-06T01:56:37,387 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1 to hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/archive/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/info/b87d1b833ab44829843bcc477bf661de.ff395bc6a25b16a53c8c3b4809bbcec1
2024-12-06T01:56:37,387 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-06T01:56:37,391 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/data/default/TestLogRolling-testLogRolling/6ce48c8d2f6a08259d933e8387571ad0/recovered.edits/128.seqid, newMaxSeqId=128, maxSeqId=123
2024-12-06T01:56:37,392 INFO  [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:56:37,392 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ce48c8d2f6a08259d933e8387571ad0:
Waiting for close lock at 1733450197384Running coprocessor pre-close hooks at 1733450197384Disabling compacts and flushes for region at 1733450197384Disabling writes for close at 1733450197385 (+1 ms)Writing region close event to WAL at 1733450197388 (+3 ms)Running coprocessor post-close hooks at 1733450197392 (+4 ms)Closed at 1733450197392
2024-12-06T01:56:37,392 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733450144539.6ce48c8d2f6a08259d933e8387571ad0.
2024-12-06T01:56:37,547 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,45189,1733450109023; all regions closed.
2024-12-06T01:56:37,548 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,548 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,548 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,548 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,548 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741834_1010 (size=8107)
2024-12-06T01:56:37,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741834_1010 (size=8107)
2024-12-06T01:56:37,553 DEBUG [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/oldWALs
2024-12-06T01:56:37,553 INFO  [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C45189%2C1733450109023.meta:.meta(num 1733450109893)
2024-12-06T01:56:37,554 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,554 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,554 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,554 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,555 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:37,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741876_1052 (size=779)
2024-12-06T01:56:37,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741876_1052 (size=779)
2024-12-06T01:56:37,559 DEBUG [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/oldWALs
2024-12-06T01:56:37,559 INFO  [RS:0;00c764f7c3aa:45189 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C45189%2C1733450109023:(num 1733450197233)
2024-12-06T01:56:37,559 DEBUG [RS:0;00c764f7c3aa:45189 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:37,559 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:56:37,559 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:56:37,559 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-06T01:56:37,559 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:56:37,559 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:56:37,559 INFO  [RS:0;00c764f7c3aa:45189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45189
2024-12-06T01:56:37,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,45189,1733450109023
2024-12-06T01:56:37,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:56:37,562 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:56:37,563 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,45189,1733450109023]
2024-12-06T01:56:37,565 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,45189,1733450109023 already deleted, retry=false
2024-12-06T01:56:37,565 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,45189,1733450109023 expired; onlineServers=0
2024-12-06T01:56:37,565 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,36107,1733450108963' *****
2024-12-06T01:56:37,565 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:56:37,565 INFO  [M:0;00c764f7c3aa:36107 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:56:37,565 INFO  [M:0;00c764f7c3aa:36107 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:56:37,565 DEBUG [M:0;00c764f7c3aa:36107 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:56:37,565 DEBUG [M:0;00c764f7c3aa:36107 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:56:37,565 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:56:37,565 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450109257 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450109257,5,FailOnTimeoutGroup]
2024-12-06T01:56:37,565 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450109263 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450109263,5,FailOnTimeoutGroup]
2024-12-06T01:56:37,565 INFO  [M:0;00c764f7c3aa:36107 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:56:37,565 INFO  [M:0;00c764f7c3aa:36107 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:56:37,565 DEBUG [M:0;00c764f7c3aa:36107 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:56:37,565 INFO  [M:0;00c764f7c3aa:36107 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:56:37,565 INFO  [M:0;00c764f7c3aa:36107 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:56:37,566 INFO  [M:0;00c764f7c3aa:36107 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:56:37,566 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:56:37,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:56:37,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:37,567 DEBUG [M:0;00c764f7c3aa:36107 {}] zookeeper.ZKUtil(347): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:56:37,567 WARN  [M:0;00c764f7c3aa:36107 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:56:37,567 INFO  [M:0;00c764f7c3aa:36107 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/.lastflushedseqids
2024-12-06T01:56:37,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741877_1053 (size=228)
2024-12-06T01:56:37,575 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741877_1053 (size=228)
2024-12-06T01:56:37,576 INFO  [M:0;00c764f7c3aa:36107 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:56:37,576 INFO  [M:0;00c764f7c3aa:36107 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:56:37,576 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:56:37,576 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:37,576 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:37,576 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:56:37,576 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:37,576 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.41 KB heapSize=63.33 KB
2024-12-06T01:56:37,600 DEBUG [M:0;00c764f7c3aa:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cb83a030109409d8e70b8cf28344b78 is 82, key is hbase:meta,,1/info:regioninfo/1733450109959/Put/seqid=0
2024-12-06T01:56:37,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741878_1054 (size=5672)
2024-12-06T01:56:37,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741878_1054 (size=5672)
2024-12-06T01:56:37,606 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cb83a030109409d8e70b8cf28344b78
2024-12-06T01:56:37,635 DEBUG [M:0;00c764f7c3aa:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ea8c349b14245a39ebd81ad92ae34ac is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733450110440/Put/seqid=0
2024-12-06T01:56:37,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741879_1055 (size=7089)
2024-12-06T01:56:37,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741879_1055 (size=7089)
2024-12-06T01:56:37,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:37,663 INFO  [RS:0;00c764f7c3aa:45189 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:56:37,663 INFO  [RS:0;00c764f7c3aa:45189 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,45189,1733450109023; zookeeper connection closed.
2024-12-06T01:56:37,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x100650948f60001, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:37,664 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@350e91e9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@350e91e9
2024-12-06T01:56:37,664 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T01:56:38,043 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.80 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ea8c349b14245a39ebd81ad92ae34ac
2024-12-06T01:56:38,048 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4ea8c349b14245a39ebd81ad92ae34ac
2024-12-06T01:56:38,065 DEBUG [M:0;00c764f7c3aa:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6efbe139c41f4ef2a261f8829790c34f is 69, key is 00c764f7c3aa,45189,1733450109023/rs:state/1733450109292/Put/seqid=0
2024-12-06T01:56:38,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741880_1056 (size=5156)
2024-12-06T01:56:38,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741880_1056 (size=5156)
2024-12-06T01:56:38,074 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6efbe139c41f4ef2a261f8829790c34f
2024-12-06T01:56:38,097 DEBUG [M:0;00c764f7c3aa:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d31e95536860446db52483719177f2c7 is 52, key is load_balancer_on/state:d/1733450110065/Put/seqid=0
2024-12-06T01:56:38,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741881_1057 (size=5056)
2024-12-06T01:56:38,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741881_1057 (size=5056)
2024-12-06T01:56:38,119 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d31e95536860446db52483719177f2c7
2024-12-06T01:56:38,124 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cb83a030109409d8e70b8cf28344b78 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cb83a030109409d8e70b8cf28344b78
2024-12-06T01:56:38,132 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cb83a030109409d8e70b8cf28344b78, entries=8, sequenceid=125, filesize=5.5 K
2024-12-06T01:56:38,133 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ea8c349b14245a39ebd81ad92ae34ac as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ea8c349b14245a39ebd81ad92ae34ac
2024-12-06T01:56:38,141 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4ea8c349b14245a39ebd81ad92ae34ac
2024-12-06T01:56:38,142 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ea8c349b14245a39ebd81ad92ae34ac, entries=13, sequenceid=125, filesize=6.9 K
2024-12-06T01:56:38,143 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6efbe139c41f4ef2a261f8829790c34f as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6efbe139c41f4ef2a261f8829790c34f
2024-12-06T01:56:38,148 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6efbe139c41f4ef2a261f8829790c34f, entries=1, sequenceid=125, filesize=5.0 K
2024-12-06T01:56:38,149 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d31e95536860446db52483719177f2c7 as hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d31e95536860446db52483719177f2c7
2024-12-06T01:56:38,154 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43255/user/jenkins/test-data/ca18e3c4-1742-bb74-f013-1f1f8b9172db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d31e95536860446db52483719177f2c7, entries=1, sequenceid=125, filesize=4.9 K
2024-12-06T01:56:38,155 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 579ms, sequenceid=125, compaction requested=false
2024-12-06T01:56:38,160 INFO  [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:38,160 DEBUG [M:0;00c764f7c3aa:36107 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450197576Disabling compacts and flushes for region at 1733450197576Disabling writes for close at 1733450197576Obtaining lock to block concurrent updates at 1733450197576Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733450197576Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52639, getHeapSize=64784, getOffHeapSize=0, getCellsCount=148 at 1733450197577 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733450197578 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733450197578Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733450197600 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733450197600Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733450197613 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733450197635 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733450197635Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733450198048 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733450198065 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733450198065Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733450198079 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733450198096 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733450198096Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724a59f3: reopening flushed file at 1733450198124 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b7aaa36: reopening flushed file at 1733450198132 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77349ac4: reopening flushed file at 1733450198142 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c48b631: reopening flushed file at 1733450198148 (+6 ms)Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 579ms, sequenceid=125, compaction requested=false at 1733450198155 (+7 ms)Writing region close event to WAL at 1733450198160 (+5 ms)Closed at 1733450198160
2024-12-06T01:56:38,160 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:38,161 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:38,161 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:38,161 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:38,161 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:38,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32953 is added to blk_1073741830_1006 (size=61308)
2024-12-06T01:56:38,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42917 is added to blk_1073741830_1006 (size=61308)
2024-12-06T01:56:38,165 INFO  [M:0;00c764f7c3aa:36107 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:56:38,165 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:56:38,165 INFO  [M:0;00c764f7c3aa:36107 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36107
2024-12-06T01:56:38,165 INFO  [M:0;00c764f7c3aa:36107 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:56:38,268 INFO  [M:0;00c764f7c3aa:36107 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:56:38,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:38,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x100650948f60000, quorum=127.0.0.1:62741, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:38,272 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@62812bcb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:56:38,273 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5fe69f2e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:56:38,273 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:56:38,273 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6fea856d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:56:38,273 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@140f1428{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir/,STOPPED}
2024-12-06T01:56:38,275 WARN  [BP-842780030-172.17.0.2-1733450108112 heartbeating to localhost/127.0.0.1:43255 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:56:38,275 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:56:38,275 WARN  [BP-842780030-172.17.0.2-1733450108112 heartbeating to localhost/127.0.0.1:43255 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-842780030-172.17.0.2-1733450108112 (Datanode Uuid 33f0282b-1ef2-4dec-bb90-c170b358e5e9) service to localhost/127.0.0.1:43255
2024-12-06T01:56:38,275 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:56:38,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:38,276 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data4/current/BP-842780030-172.17.0.2-1733450108112 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:38,276 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:56:38,277 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data3/current/BP-842780030-172.17.0.2-1733450108112 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:38,289 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b03ccf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:56:38,289 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bc201cc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:56:38,290 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:56:38,290 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@646e9349{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:56:38,290 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36ab964c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir/,STOPPED}
2024-12-06T01:56:38,291 WARN  [BP-842780030-172.17.0.2-1733450108112 heartbeating to localhost/127.0.0.1:43255 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:56:38,291 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:56:38,291 WARN  [BP-842780030-172.17.0.2-1733450108112 heartbeating to localhost/127.0.0.1:43255 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-842780030-172.17.0.2-1733450108112 (Datanode Uuid 4af9ca99-38e6-42e0-aef4-c33476dbd857) service to localhost/127.0.0.1:43255
2024-12-06T01:56:38,291 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:56:38,292 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data1/current/BP-842780030-172.17.0.2-1733450108112 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:38,292 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/cluster_7cd9430f-c2f5-9968-30dc-9aa594f033c1/data/data2/current/BP-842780030-172.17.0.2-1733450108112 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:38,293 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:56:38,302 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b14e4d9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:56:38,302 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2424cbbd{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:56:38,302 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:56:38,302 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10fad9f2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:56:38,303 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5eef5514{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir/,STOPPED}
2024-12-06T01:56:38,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:38,313 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:56:38,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:56:38,362 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=228 (was 204)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43255
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/00c764f7c3aa:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:43255 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43255
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:43255
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:43255 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:43255 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=506 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 291) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8930 (was 9193)
2024-12-06T01:56:38,373 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=228, OpenFileDescriptor=506, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=8928
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.log.dir so I do NOT create it in target/test-data/f895c614-2311-731d-2a65-1f697f071225
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86f4ce7a-907f-e360-4b2d-d1ebd4085d30/hadoop.tmp.dir so I do NOT create it in target/test-data/f895c614-2311-731d-2a65-1f697f071225
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf, deleteOnExit=true
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/test.cache.data in system properties and HBase conf
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.tmp.dir in system properties and HBase conf
2024-12-06T01:56:38,374 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir in system properties and HBase conf
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-06T01:56:38,375 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-06T01:56:38,375 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/nfs.dump.dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/java.io.tmpdir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-06T01:56:38,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-06T01:56:38,392 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:56:38,456 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:56:38,460 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:56:38,462 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:56:38,462 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:56:38,462 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-06T01:56:38,463 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:56:38,463 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15cc83a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:56:38,463 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@322729b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:56:38,592 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65b573aa{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/java.io.tmpdir/jetty-localhost-39759-hadoop-hdfs-3_4_1-tests_jar-_-any-12024177247554671135/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:56:38,592 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@50345f99{HTTP/1.1, (http/1.1)}{localhost:39759}
2024-12-06T01:56:38,592 INFO  [Time-limited test {}] server.Server(415): Started @331860ms
2024-12-06T01:56:38,605 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-06T01:56:38,662 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:56:38,666 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:56:38,667 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:56:38,667 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:56:38,667 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:56:38,667 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26564ea0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:56:38,668 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e06ed3c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:56:38,799 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fd2bdde{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/java.io.tmpdir/jetty-localhost-41081-hadoop-hdfs-3_4_1-tests_jar-_-any-2100835702877645021/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:56:38,799 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d26a1a7{HTTP/1.1, (http/1.1)}{localhost:41081}
2024-12-06T01:56:38,799 INFO  [Time-limited test {}] server.Server(415): Started @332067ms
2024-12-06T01:56:38,801 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:56:38,837 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-06T01:56:38,840 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-06T01:56:38,841 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-06T01:56:38,841 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-06T01:56:38,841 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-06T01:56:38,841 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20c7d165{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir/,AVAILABLE}
2024-12-06T01:56:38,842 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3687b779{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-06T01:56:38,900 WARN  [Thread-2480 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data1/current/BP-450157059-172.17.0.2-1733450198398/current, will proceed with Du for space computation calculation, 
2024-12-06T01:56:38,900 WARN  [Thread-2481 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data2/current/BP-450157059-172.17.0.2-1733450198398/current, will proceed with Du for space computation calculation, 
2024-12-06T01:56:38,925 WARN  [Thread-2459 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:56:38,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x62b5d10944ea4a2f with lease ID 0x697969da6023d5e8: Processing first storage report for DS-fce240e0-622f-4eaa-931d-74d1423105a3 from datanode DatanodeRegistration(127.0.0.1:42257, datanodeUuid=c846784d-567f-4498-9227-66696fb12e44, infoPort=39331, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398)
2024-12-06T01:56:38,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x62b5d10944ea4a2f with lease ID 0x697969da6023d5e8: from storage DS-fce240e0-622f-4eaa-931d-74d1423105a3 node DatanodeRegistration(127.0.0.1:42257, datanodeUuid=c846784d-567f-4498-9227-66696fb12e44, infoPort=39331, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:56:38,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x62b5d10944ea4a2f with lease ID 0x697969da6023d5e8: Processing first storage report for DS-d2b51810-038f-494f-9da1-790292987e88 from datanode DatanodeRegistration(127.0.0.1:42257, datanodeUuid=c846784d-567f-4498-9227-66696fb12e44, infoPort=39331, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398)
2024-12-06T01:56:38,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x62b5d10944ea4a2f with lease ID 0x697969da6023d5e8: from storage DS-d2b51810-038f-494f-9da1-790292987e88 node DatanodeRegistration(127.0.0.1:42257, datanodeUuid=c846784d-567f-4498-9227-66696fb12e44, infoPort=39331, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:56:38,973 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d5d1a79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/java.io.tmpdir/jetty-localhost-39955-hadoop-hdfs-3_4_1-tests_jar-_-any-16517021924101437928/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:56:38,973 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ebefcd5{HTTP/1.1, (http/1.1)}{localhost:39955}
2024-12-06T01:56:38,973 INFO  [Time-limited test {}] server.Server(415): Started @332241ms
2024-12-06T01:56:38,974 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-06T01:56:39,064 WARN  [Thread-2506 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data3/current/BP-450157059-172.17.0.2-1733450198398/current, will proceed with Du for space computation calculation, 
2024-12-06T01:56:39,064 WARN  [Thread-2507 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data4/current/BP-450157059-172.17.0.2-1733450198398/current, will proceed with Du for space computation calculation, 
2024-12-06T01:56:39,089 WARN  [Thread-2495 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-06T01:56:39,092 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba0286a08b8a1fb6 with lease ID 0x697969da6023d5e9: Processing first storage report for DS-ceabc13d-80cd-4b7c-a6a8-1149bcc83363 from datanode DatanodeRegistration(127.0.0.1:38129, datanodeUuid=d2b354d5-8e1b-4720-a780-4b097bb9895b, infoPort=33603, infoSecurePort=0, ipcPort=42317, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398)
2024-12-06T01:56:39,092 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba0286a08b8a1fb6 with lease ID 0x697969da6023d5e9: from storage DS-ceabc13d-80cd-4b7c-a6a8-1149bcc83363 node DatanodeRegistration(127.0.0.1:38129, datanodeUuid=d2b354d5-8e1b-4720-a780-4b097bb9895b, infoPort=33603, infoSecurePort=0, ipcPort=42317, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-06T01:56:39,092 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba0286a08b8a1fb6 with lease ID 0x697969da6023d5e9: Processing first storage report for DS-9744d28d-25f3-43e7-a365-3adf6a17eaf6 from datanode DatanodeRegistration(127.0.0.1:38129, datanodeUuid=d2b354d5-8e1b-4720-a780-4b097bb9895b, infoPort=33603, infoSecurePort=0, ipcPort=42317, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398)
2024-12-06T01:56:39,092 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba0286a08b8a1fb6 with lease ID 0x697969da6023d5e9: from storage DS-9744d28d-25f3-43e7-a365-3adf6a17eaf6 node DatanodeRegistration(127.0.0.1:38129, datanodeUuid=d2b354d5-8e1b-4720-a780-4b097bb9895b, infoPort=33603, infoSecurePort=0, ipcPort=42317, storageInfo=lv=-57;cid=testClusterID;nsid=441020897;c=1733450198398), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-06T01:56:39,097 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225
2024-12-06T01:56:39,104 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/zookeeper_0, clientPort=61840, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-06T01:56:39,105 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61840
2024-12-06T01:56:39,106 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,107 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:56:39,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741825_1001 (size=7)
2024-12-06T01:56:39,135 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35 with version=8
2024-12-06T01:56:39,135 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36221/user/jenkins/test-data/ac0f9f55-f6ce-9b44-b562-2919c3ecfa18/hbase-staging
2024-12-06T01:56:39,137 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-06T01:56:39,138 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:56:39,139 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34355
2024-12-06T01:56:39,140 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34355 connecting to ZooKeeper ensemble=127.0.0.1:61840
2024-12-06T01:56:39,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:343550x0, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:56:39,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34355-0x100650aa9220000 connected
2024-12-06T01:56:39,169 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,170 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,172 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:56:39,172 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35, hbase.cluster.distributed=false
2024-12-06T01:56:39,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:56:39,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34355
2024-12-06T01:56:39,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34355
2024-12-06T01:56:39,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34355
2024-12-06T01:56:39,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34355
2024-12-06T01:56:39,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34355
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-06T01:56:39,193 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-06T01:56:39,194 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42727
2024-12-06T01:56:39,195 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42727 connecting to ZooKeeper ensemble=127.0.0.1:61840
2024-12-06T01:56:39,195 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,197 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427270x0, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-06T01:56:39,202 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:427270x0, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:56:39,202 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42727-0x100650aa9220001 connected
2024-12-06T01:56:39,202 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-06T01:56:39,202 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-06T01:56:39,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-06T01:56:39,204 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-06T01:56:39,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42727
2024-12-06T01:56:39,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42727
2024-12-06T01:56:39,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42727
2024-12-06T01:56:39,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42727
2024-12-06T01:56:39,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42727
2024-12-06T01:56:39,223 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:34355
2024-12-06T01:56:39,223 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:56:39,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:56:39,225 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-06T01:56:39,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,228 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-06T01:56:39,228 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,34355,1733450199137 from backup master directory
2024-12-06T01:56:39,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:56:39,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-06T01:56:39,230 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:56:39,230 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/hbase.id] with ID: 2bf564c5-97f0-40d4-8bd5-86fe5af2cd43
2024-12-06T01:56:39,234 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/.tmp/hbase.id
2024-12-06T01:56:39,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:56:39,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741826_1002 (size=42)
2024-12-06T01:56:39,256 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/.tmp/hbase.id]:[hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/hbase.id]
2024-12-06T01:56:39,268 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:39,269 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-06T01:56:39,270 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-06T01:56:39,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:39,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:56:39,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741827_1003 (size=196)
2024-12-06T01:56:39,284 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-06T01:56:39,285 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-06T01:56:39,285 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:56:39,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:56:39,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741828_1004 (size=1189)
2024-12-06T01:56:39,293 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store
2024-12-06T01:56:39,301 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:56:39,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741829_1005 (size=34)
2024-12-06T01:56:39,302 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:56:39,302 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:56:39,302 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:39,302 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:39,302 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:56:39,302 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:39,302 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:39,302 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450199302Disabling compacts and flushes for region at 1733450199302Disabling writes for close at 1733450199302Writing region close event to WAL at 1733450199302Closed at 1733450199302
2024-12-06T01:56:39,303 WARN  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/.initializing
2024-12-06T01:56:39,303 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/WALs/00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,306 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C34355%2C1733450199137, suffix=, logDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/WALs/00c764f7c3aa,34355,1733450199137, archiveDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/oldWALs, maxLogs=10
2024-12-06T01:56:39,307 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C34355%2C1733450199137.1733450199306
2024-12-06T01:56:39,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:39,313 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/WALs/00c764f7c3aa,34355,1733450199137/00c764f7c3aa%2C34355%2C1733450199137.1733450199306
2024-12-06T01:56:39,314 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39331:39331),(127.0.0.1/127.0.0.1:33603:33603)]
2024-12-06T01:56:39,314 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:56:39,315 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:56:39,315 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,315 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,316 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:56:39,317 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-06T01:56:39,318 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:39,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:56:39,320 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-06T01:56:39,320 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,320 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:56:39,320 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:56:39,322 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-06T01:56:39,322 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,322 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:56:39,322 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-06T01:56:39,324 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-06T01:56:39,324 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,324 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-06T01:56:39,324 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,325 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,326 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,327 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,327 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,328 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-06T01:56:39,328 INFO  [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:56:39,329 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-06T01:56:39,332 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:56:39,333 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850308, jitterRate=0.08122356235980988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-06T01:56:39,334 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733450199315Initializing all the Stores at 1733450199316 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450199316Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450199316Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450199316Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450199316Cleaning up temporary data from old regions at 1733450199327 (+11 ms)Region opened successfully at 1733450199333 (+6 ms)
2024-12-06T01:56:39,334 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-06T01:56:39,341 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5613cd0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:56:39,342 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-06T01:56:39,342 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-06T01:56:39,342 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-06T01:56:39,342 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-06T01:56:39,343 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-06T01:56:39,343 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-06T01:56:39,343 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-06T01:56:39,345 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-06T01:56:39,346 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-06T01:56:39,347 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-06T01:56:39,348 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-06T01:56:39,348 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-06T01:56:39,349 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-06T01:56:39,350 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-06T01:56:39,351 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-06T01:56:39,352 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-06T01:56:39,353 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-06T01:56:39,354 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-06T01:56:39,356 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-06T01:56:39,357 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-06T01:56:39,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:56:39,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-06T01:56:39,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,359 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,34355,1733450199137, sessionid=0x100650aa9220000, setting cluster-up flag (Was=false)
2024-12-06T01:56:39,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,368 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-06T01:56:39,369 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,377 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-06T01:56:39,378 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:39,380 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-06T01:56:39,381 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-06T01:56:39,382 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-06T01:56:39,382 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-06T01:56:39,382 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,34355,1733450199137
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-06T01:56:39,383 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:56:39,384 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,387 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:56:39,387 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-06T01:56:39,389 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,389 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733450229391
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-06T01:56:39,391 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,392 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-06T01:56:39,392 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-06T01:56:39,392 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-06T01:56:39,393 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-06T01:56:39,393 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-06T01:56:39,396 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450199393,5,FailOnTimeoutGroup]
2024-12-06T01:56:39,396 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450199396,5,FailOnTimeoutGroup]
2024-12-06T01:56:39,396 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,396 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-06T01:56:39,396 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,396 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:56:39,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741831_1007 (size=1321)
2024-12-06T01:56:39,400 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-06T01:56:39,400 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35
2024-12-06T01:56:39,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:56:39,415 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(746): ClusterId : 2bf564c5-97f0-40d4-8bd5-86fe5af2cd43
2024-12-06T01:56:39,415 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-06T01:56:39,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741832_1008 (size=32)
2024-12-06T01:56:39,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:56:39,418 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:56:39,418 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-06T01:56:39,418 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-06T01:56:39,420 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:56:39,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,420 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:39,420 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:56:39,420 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-06T01:56:39,421 DEBUG [RS:0;00c764f7c3aa:42727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2782d3f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0
2024-12-06T01:56:39,422 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:56:39,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,422 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:39,422 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:56:39,424 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:56:39,424 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,424 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:39,424 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:56:39,426 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:56:39,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:39,426 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:39,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:56:39,427 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740
2024-12-06T01:56:39,427 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740
2024-12-06T01:56:39,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:56:39,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:56:39,429 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:56:39,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:56:39,433 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-06T01:56:39,434 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=700966, jitterRate=-0.10867667198181152}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:56:39,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733450199417Initializing all the Stores at 1733450199417Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450199418 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450199418Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450199418Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450199418Cleaning up temporary data from old regions at 1733450199429 (+11 ms)Region opened successfully at 1733450199434 (+5 ms)
2024-12-06T01:56:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:56:39,435 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:56:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:56:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:56:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:56:39,435 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:56:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450199435Disabling compacts and flushes for region at 1733450199435Disabling writes for close at 1733450199435Writing region close event to WAL at 1733450199435Closed at 1733450199435
2024-12-06T01:56:39,437 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:42727
2024-12-06T01:56:39,437 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:56:39,437 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-06T01:56:39,437 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-06T01:56:39,437 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-06T01:56:39,437 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-06T01:56:39,437 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-06T01:56:39,438 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,34355,1733450199137 with port=42727, startcode=1733450199192
2024-12-06T01:56:39,438 DEBUG [RS:0;00c764f7c3aa:42727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-06T01:56:39,439 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:56:39,440 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-06T01:56:39,441 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38599, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-06T01:56:39,441 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,441 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34355 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,443 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35
2024-12-06T01:56:39,443 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43593
2024-12-06T01:56:39,443 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-06T01:56:39,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:56:39,445 DEBUG [RS:0;00c764f7c3aa:42727 {}] zookeeper.ZKUtil(111): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,445 WARN  [RS:0;00c764f7c3aa:42727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-06T01:56:39,445 INFO  [RS:0;00c764f7c3aa:42727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:56:39,445 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,445 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,42727,1733450199192]
2024-12-06T01:56:39,449 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-06T01:56:39,450 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-06T01:56:39,451 INFO  [RS:0;00c764f7c3aa:42727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-06T01:56:39,451 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,451 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-06T01:56:39,452 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-06T01:56:39,452 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2
2024-12-06T01:56:39,452 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:56:39,453 DEBUG [RS:0;00c764f7c3aa:42727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3
2024-12-06T01:56:39,453 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,453 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,453 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,453 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,453 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,454 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42727,1733450199192-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:56:39,476 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-06T01:56:39,476 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,42727,1733450199192-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,476 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,476 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.Replication(171): 00c764f7c3aa,42727,1733450199192 started
2024-12-06T01:56:39,498 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:39,498 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,42727,1733450199192, RpcServer on 00c764f7c3aa/172.17.0.2:42727, sessionid=0x100650aa9220001
2024-12-06T01:56:39,498 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-06T01:56:39,498 DEBUG [RS:0;00c764f7c3aa:42727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,498 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,42727,1733450199192'
2024-12-06T01:56:39,498 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-06T01:56:39,499 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-06T01:56:39,499 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-06T01:56:39,499 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-06T01:56:39,500 DEBUG [RS:0;00c764f7c3aa:42727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,500 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,42727,1733450199192'
2024-12-06T01:56:39,500 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-06T01:56:39,500 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-06T01:56:39,500 DEBUG [RS:0;00c764f7c3aa:42727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-06T01:56:39,500 INFO  [RS:0;00c764f7c3aa:42727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-06T01:56:39,500 INFO  [RS:0;00c764f7c3aa:42727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-06T01:56:39,590 WARN  [00c764f7c3aa:34355 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-06T01:56:39,602 INFO  [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C42727%2C1733450199192, suffix=, logDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/00c764f7c3aa,42727,1733450199192, archiveDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs, maxLogs=32
2024-12-06T01:56:39,618 INFO  [RS:0;00c764f7c3aa:42727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42727%2C1733450199192.1733450199617
2024-12-06T01:56:39,624 INFO  [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/00c764f7c3aa,42727,1733450199192/00c764f7c3aa%2C42727%2C1733450199192.1733450199617
2024-12-06T01:56:39,625 DEBUG [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33603:33603),(127.0.0.1/127.0.0.1:39331:39331)]
2024-12-06T01:56:39,840 DEBUG [00c764f7c3aa:34355 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-06T01:56:39,841 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:39,842 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,42727,1733450199192, state=OPENING
2024-12-06T01:56:39,844 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-06T01:56:39,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:39,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:56:39,846 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:56:39,846 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-06T01:56:39,846 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,42727,1733450199192}]
2024-12-06T01:56:40,000 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-06T01:56:40,002 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47285, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-06T01:56:40,005 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-06T01:56:40,006 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:56:40,007 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C42727%2C1733450199192.meta, suffix=.meta, logDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/00c764f7c3aa,42727,1733450199192, archiveDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs, maxLogs=32
2024-12-06T01:56:40,008 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 00c764f7c3aa%2C42727%2C1733450199192.meta.1733450200008.meta
2024-12-06T01:56:40,030 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/00c764f7c3aa,42727,1733450199192/00c764f7c3aa%2C42727%2C1733450199192.meta.1733450200008.meta
2024-12-06T01:56:40,032 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33603:33603),(127.0.0.1/127.0.0.1:39331:39331)]
2024-12-06T01:56:40,033 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-06T01:56:40,033 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-06T01:56:40,034 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-06T01:56:40,034 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-06T01:56:40,034 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-06T01:56:40,034 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-06T01:56:40,034 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-06T01:56:40,034 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-06T01:56:40,035 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-06T01:56:40,036 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-06T01:56:40,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:40,036 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:40,037 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-06T01:56:40,037 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-06T01:56:40,037 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:40,038 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:40,038 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-06T01:56:40,038 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-06T01:56:40,038 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:40,039 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:40,039 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-06T01:56:40,040 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-06T01:56:40,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-06T01:56:40,041 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-06T01:56:40,041 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-06T01:56:40,041 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740
2024-12-06T01:56:40,042 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740
2024-12-06T01:56:40,043 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-06T01:56:40,043 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-06T01:56:40,044 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-06T01:56:40,045 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-06T01:56:40,046 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=875120, jitterRate=0.1127738356590271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-06T01:56:40,046 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-06T01:56:40,047 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733450200034Writing region info on filesystem at 1733450200034Initializing all the Stores at 1733450200035 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450200035Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450200035Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733450200035Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733450200035Cleaning up temporary data from old regions at 1733450200043 (+8 ms)Running coprocessor post-open hooks at 1733450200046 (+3 ms)Region opened successfully at 1733450200047 (+1 ms)
2024-12-06T01:56:40,048 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733450199999
2024-12-06T01:56:40,050 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-06T01:56:40,050 INFO  [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-06T01:56:40,051 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:40,052 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,42727,1733450199192, state=OPEN
2024-12-06T01:56:40,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:56:40,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-06T01:56:40,057 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:40,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:56:40,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-06T01:56:40,059 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-06T01:56:40,059 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,42727,1733450199192 in 211 msec
2024-12-06T01:56:40,061 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-06T01:56:40,061 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 622 msec
2024-12-06T01:56:40,062 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-06T01:56:40,062 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-06T01:56:40,063 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:56:40,064 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,42727,1733450199192, seqNum=-1]
2024-12-06T01:56:40,064 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:56:40,065 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60023, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:56:40,070 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 688 msec
2024-12-06T01:56:40,070 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733450200070, completionTime=-1
2024-12-06T01:56:40,070 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-06T01:56:40,070 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-06T01:56:40,072 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-06T01:56:40,072 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733450260072
2024-12-06T01:56:40,072 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733450320072
2024-12-06T01:56:40,072 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-06T01:56:40,073 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34355,1733450199137-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,073 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34355,1733450199137-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,073 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34355,1733450199137-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,073 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:34355, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,073 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,073 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,074 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.846sec
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34355,1733450199137-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-06T01:56:40,077 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34355,1733450199137-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-06T01:56:40,079 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-06T01:56:40,079 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-06T01:56:40,080 INFO  [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,34355,1733450199137-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-06T01:56:40,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a32861f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:56:40,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,34355,-1 for getting cluster id
2024-12-06T01:56:40,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-06T01:56:40,121 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2bf564c5-97f0-40d4-8bd5-86fe5af2cd43'
2024-12-06T01:56:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-06T01:56:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2bf564c5-97f0-40d4-8bd5-86fe5af2cd43"

2024-12-06T01:56:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27707685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:56:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,34355,-1]
2024-12-06T01:56:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-06T01:56:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:40,123 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-06T01:56:40,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3597b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-06T01:56:40,124 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-06T01:56:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,42727,1733450199192, seqNum=-1]
2024-12-06T01:56:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-06T01:56:40,126 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-06T01:56:40,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:40,129 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-06T01:56:40,131 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-06T01:56:40,131 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-06T01:56:40,134 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/test.com,8080,1, archiveDir=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs, maxLogs=32
2024-12-06T01:56:40,134 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733450200134
2024-12-06T01:56:40,140 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/test.com,8080,1/test.com%2C8080%2C1.1733450200134
2024-12-06T01:56:40,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39331:39331),(127.0.0.1/127.0.0.1:33603:33603)]
2024-12-06T01:56:40,146 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733450200146
2024-12-06T01:56:40,153 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,153 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,153 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,153 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,153 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,153 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/test.com,8080,1/test.com%2C8080%2C1.1733450200134 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/test.com,8080,1/test.com%2C8080%2C1.1733450200146
2024-12-06T01:56:40,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741835_1011 (size=93)
2024-12-06T01:56:40,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741835_1011 (size=93)
2024-12-06T01:56:40,160 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/WALs/test.com,8080,1/test.com%2C8080%2C1.1733450200134 to hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs/test.com%2C8080%2C1.1733450200134
2024-12-06T01:56:40,160 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33603:33603),(127.0.0.1/127.0.0.1:39331:39331)]
2024-12-06T01:56:40,161 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,161 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,161 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,161 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,161 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741836_1012 (size=93)
2024-12-06T01:56:40,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741836_1012 (size=93)
2024-12-06T01:56:40,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs
2024-12-06T01:56:40,166 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733450200146)
2024-12-06T01:56:40,166 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-06T01:56:40,167 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:56:40,167 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:56:40,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:40,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:40,167 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-06T01:56:40,167 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-06T01:56:40,167 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=166253697, stopped=false
2024-12-06T01:56:40,167 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,34355,1733450199137
2024-12-06T01:56:40,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:56:40,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:40,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-06T01:56:40,169 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:56:40,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:40,169 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-06T01:56:40,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:56:40,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:40,169 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,42727,1733450199192' *****
2024-12-06T01:56:40,169 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-06T01:56:40,170 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-06T01:56:40,170 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:42727.
2024-12-06T01:56:40,170 DEBUG [RS:0;00c764f7c3aa:42727 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-06T01:56:40,170 DEBUG [RS:0;00c764f7c3aa:42727 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-06T01:56:40,170 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-06T01:56:40,170 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-06T01:56:40,171 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-06T01:56:40,171 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-06T01:56:40,171 DEBUG [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-06T01:56:40,171 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-06T01:56:40,171 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-06T01:56:40,171 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-06T01:56:40,171 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-06T01:56:40,171 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-06T01:56:40,171 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-06T01:56:40,189 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/.tmp/ns/3fc9cc29e37d4b25924b53591095a688 is 43, key is default/ns:d/1733450200066/Put/seqid=0
2024-12-06T01:56:40,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741837_1013 (size=5153)
2024-12-06T01:56:40,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741837_1013 (size=5153)
2024-12-06T01:56:40,195 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/.tmp/ns/3fc9cc29e37d4b25924b53591095a688
2024-12-06T01:56:40,201 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/.tmp/ns/3fc9cc29e37d4b25924b53591095a688 as hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/ns/3fc9cc29e37d4b25924b53591095a688
2024-12-06T01:56:40,205 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/ns/3fc9cc29e37d4b25924b53591095a688, entries=2, sequenceid=6, filesize=5.0 K
2024-12-06T01:56:40,206 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=6, compaction requested=false
2024-12-06T01:56:40,207 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-06T01:56:40,211 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-06T01:56:40,211 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-06T01:56:40,211 INFO  [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-06T01:56:40,212 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733450200171Running coprocessor pre-close hooks at 1733450200171Disabling compacts and flushes for region at 1733450200171Disabling writes for close at 1733450200171Obtaining lock to block concurrent updates at 1733450200171Preparing flush snapshotting stores in 1588230740 at 1733450200171Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733450200172 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733450200172Flushing 1588230740/ns: creating writer at 1733450200173 (+1 ms)Flushing 1588230740/ns: appending metadata at 1733450200189 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733450200189Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aad1fd1: reopening flushed file at 1733450200200 (+11 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=6, compaction requested=false at 1733450200206 (+6 ms)Writing region close event to WAL at 1733450200208 (+2 ms)Running coprocessor post-close hooks at 1733450200211 (+3 ms)Closed at 1733450200211
2024-12-06T01:56:40,212 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-06T01:56:40,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,37779,1733449974953/00c764f7c3aa%2C37779%2C1733449974953.1733449975140
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:40,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41913/user/jenkins/test-data/f6a1e930-4244-6f82-ad23-47ae7d224f4c/WALs/00c764f7c3aa,45229,1733449973578/00c764f7c3aa%2C45229%2C1733449973578.meta.1733449974801.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-06T01:56:40,371 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,42727,1733450199192; all regions closed.
2024-12-06T01:56:40,371 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,372 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,372 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,372 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,372 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741834_1010 (size=1152)
2024-12-06T01:56:40,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741834_1010 (size=1152)
2024-12-06T01:56:40,377 DEBUG [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs
2024-12-06T01:56:40,377 INFO  [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C42727%2C1733450199192.meta:.meta(num 1733450200008)
2024-12-06T01:56:40,377 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,378 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,378 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,378 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,378 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741833_1009 (size=93)
2024-12-06T01:56:40,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741833_1009 (size=93)
2024-12-06T01:56:40,382 DEBUG [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/oldWALs
2024-12-06T01:56:40,382 INFO  [RS:0;00c764f7c3aa:42727 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 00c764f7c3aa%2C42727%2C1733450199192:(num 1733450199617)
2024-12-06T01:56:40,382 DEBUG [RS:0;00c764f7c3aa:42727 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-06T01:56:40,382 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.LeaseManager(133): Closed leases
2024-12-06T01:56:40,382 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:56:40,383 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-06T01:56:40,383 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:56:40,383 INFO  [RS:0;00c764f7c3aa:42727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42727
2024-12-06T01:56:40,383 INFO  [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:56:40,385 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:56:40,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,42727,1733450199192
2024-12-06T01:56:40,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-06T01:56:40,387 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,42727,1733450199192]
2024-12-06T01:56:40,388 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,42727,1733450199192 already deleted, retry=false
2024-12-06T01:56:40,388 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,42727,1733450199192 expired; onlineServers=0
2024-12-06T01:56:40,388 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,34355,1733450199137' *****
2024-12-06T01:56:40,388 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-06T01:56:40,388 INFO  [M:0;00c764f7c3aa:34355 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-06T01:56:40,388 INFO  [M:0;00c764f7c3aa:34355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-06T01:56:40,388 DEBUG [M:0;00c764f7c3aa:34355 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-06T01:56:40,389 DEBUG [M:0;00c764f7c3aa:34355 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-06T01:56:40,389 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450199393 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733450199393,5,FailOnTimeoutGroup]
2024-12-06T01:56:40,389 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450199396 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733450199396,5,FailOnTimeoutGroup]
2024-12-06T01:56:40,389 INFO  [M:0;00c764f7c3aa:34355 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-06T01:56:40,389 INFO  [M:0;00c764f7c3aa:34355 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-06T01:56:40,389 DEBUG [M:0;00c764f7c3aa:34355 {}] master.HMaster(1795): Stopping service threads
2024-12-06T01:56:40,389 INFO  [M:0;00c764f7c3aa:34355 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-06T01:56:40,389 INFO  [M:0;00c764f7c3aa:34355 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-06T01:56:40,389 INFO  [M:0;00c764f7c3aa:34355 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-06T01:56:40,389 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-06T01:56:40,390 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-06T01:56:40,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-06T01:56:40,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-06T01:56:40,391 DEBUG [M:0;00c764f7c3aa:34355 {}] zookeeper.ZKUtil(347): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-06T01:56:40,392 WARN  [M:0;00c764f7c3aa:34355 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-06T01:56:40,392 INFO  [M:0;00c764f7c3aa:34355 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/.lastflushedseqids
2024-12-06T01:56:40,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741838_1014 (size=99)
2024-12-06T01:56:40,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741838_1014 (size=99)
2024-12-06T01:56:40,404 INFO  [M:0;00c764f7c3aa:34355 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-06T01:56:40,405 INFO  [M:0;00c764f7c3aa:34355 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-06T01:56:40,405 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-06T01:56:40,405 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:40,405 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:40,405 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-06T01:56:40,405 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:40,405 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-06T01:56:40,422 DEBUG [M:0;00c764f7c3aa:34355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1d4e959d46845d2b1518a1607af8598 is 82, key is hbase:meta,,1/info:regioninfo/1733450200051/Put/seqid=0
2024-12-06T01:56:40,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741839_1015 (size=5672)
2024-12-06T01:56:40,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741839_1015 (size=5672)
2024-12-06T01:56:40,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:40,487 INFO  [RS:0;00c764f7c3aa:42727 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:56:40,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42727-0x100650aa9220001, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:40,487 INFO  [RS:0;00c764f7c3aa:42727 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,42727,1733450199192; zookeeper connection closed.
2024-12-06T01:56:40,487 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b2f5b82 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b2f5b82
2024-12-06T01:56:40,487 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-06T01:56:40,831 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1d4e959d46845d2b1518a1607af8598
2024-12-06T01:56:40,853 DEBUG [M:0;00c764f7c3aa:34355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad5f1428d1734eb892f702caf4b4f140 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733450200069/Put/seqid=0
2024-12-06T01:56:40,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741840_1016 (size=5275)
2024-12-06T01:56:40,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741840_1016 (size=5275)
2024-12-06T01:56:40,862 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad5f1428d1734eb892f702caf4b4f140
2024-12-06T01:56:40,883 DEBUG [M:0;00c764f7c3aa:34355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0daf00a1818e4beba662645d7593a763 is 69, key is 00c764f7c3aa,42727,1733450199192/rs:state/1733450199442/Put/seqid=0
2024-12-06T01:56:40,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741841_1017 (size=5156)
2024-12-06T01:56:40,895 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741841_1017 (size=5156)
2024-12-06T01:56:40,896 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0daf00a1818e4beba662645d7593a763
2024-12-06T01:56:40,921 DEBUG [M:0;00c764f7c3aa:34355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/be1ea12cd44543a0a72ddfc1fd153824 is 52, key is load_balancer_on/state:d/1733450200130/Put/seqid=0
2024-12-06T01:56:40,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741842_1018 (size=5056)
2024-12-06T01:56:40,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741842_1018 (size=5056)
2024-12-06T01:56:40,927 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/be1ea12cd44543a0a72ddfc1fd153824
2024-12-06T01:56:40,932 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1d4e959d46845d2b1518a1607af8598 as hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1d4e959d46845d2b1518a1607af8598
2024-12-06T01:56:40,938 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1d4e959d46845d2b1518a1607af8598, entries=8, sequenceid=29, filesize=5.5 K
2024-12-06T01:56:40,938 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad5f1428d1734eb892f702caf4b4f140 as hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad5f1428d1734eb892f702caf4b4f140
2024-12-06T01:56:40,943 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad5f1428d1734eb892f702caf4b4f140, entries=3, sequenceid=29, filesize=5.2 K
2024-12-06T01:56:40,944 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0daf00a1818e4beba662645d7593a763 as hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0daf00a1818e4beba662645d7593a763
2024-12-06T01:56:40,949 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0daf00a1818e4beba662645d7593a763, entries=1, sequenceid=29, filesize=5.0 K
2024-12-06T01:56:40,950 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/be1ea12cd44543a0a72ddfc1fd153824 as hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/be1ea12cd44543a0a72ddfc1fd153824
2024-12-06T01:56:40,955 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43593/user/jenkins/test-data/c773a53f-66a1-d233-d43a-e745ee9aba35/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/be1ea12cd44543a0a72ddfc1fd153824, entries=1, sequenceid=29, filesize=4.9 K
2024-12-06T01:56:40,957 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 552ms, sequenceid=29, compaction requested=false
2024-12-06T01:56:40,959 INFO  [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-06T01:56:40,959 DEBUG [M:0;00c764f7c3aa:34355 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733450200405Disabling compacts and flushes for region at 1733450200405Disabling writes for close at 1733450200405Obtaining lock to block concurrent updates at 1733450200405Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733450200405Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733450200405Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733450200406 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733450200406Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733450200422 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733450200422Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733450200837 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733450200852 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733450200852Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733450200867 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733450200882 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733450200882Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733450200901 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733450200921 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733450200921Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9995f0: reopening flushed file at 1733450200931 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5457afaa: reopening flushed file at 1733450200938 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be2850f: reopening flushed file at 1733450200943 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba67749: reopening flushed file at 1733450200949 (+6 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 552ms, sequenceid=29, compaction requested=false at 1733450200957 (+8 ms)Writing region close event to WAL at 1733450200959 (+2 ms)Closed at 1733450200959
2024-12-06T01:56:40,959 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,960 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,960 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,961 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,961 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-06T01:56:40,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38129 is added to blk_1073741830_1006 (size=10311)
2024-12-06T01:56:40,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42257 is added to blk_1073741830_1006 (size=10311)
2024-12-06T01:56:40,964 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-06T01:56:40,964 INFO  [M:0;00c764f7c3aa:34355 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-06T01:56:40,964 INFO  [M:0;00c764f7c3aa:34355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34355
2024-12-06T01:56:40,964 INFO  [M:0;00c764f7c3aa:34355 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-06T01:56:41,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:41,066 INFO  [M:0;00c764f7c3aa:34355 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-06T01:56:41,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34355-0x100650aa9220000, quorum=127.0.0.1:61840, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-06T01:56:41,069 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d5d1a79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:56:41,069 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ebefcd5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:56:41,069 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:56:41,069 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3687b779{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:56:41,070 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20c7d165{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir/,STOPPED}
2024-12-06T01:56:41,071 WARN  [BP-450157059-172.17.0.2-1733450198398 heartbeating to localhost/127.0.0.1:43593 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:56:41,071 WARN  [BP-450157059-172.17.0.2-1733450198398 heartbeating to localhost/127.0.0.1:43593 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-450157059-172.17.0.2-1733450198398 (Datanode Uuid d2b354d5-8e1b-4720-a780-4b097bb9895b) service to localhost/127.0.0.1:43593
2024-12-06T01:56:41,072 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:56:41,072 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:56:41,072 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data3/current/BP-450157059-172.17.0.2-1733450198398 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:41,072 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data4/current/BP-450157059-172.17.0.2-1733450198398 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:41,073 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:56:41,075 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fd2bdde{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-06T01:56:41,075 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d26a1a7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:56:41,075 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:56:41,075 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e06ed3c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:56:41,075 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26564ea0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir/,STOPPED}
2024-12-06T01:56:41,077 WARN  [BP-450157059-172.17.0.2-1733450198398 heartbeating to localhost/127.0.0.1:43593 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-06T01:56:41,077 WARN  [BP-450157059-172.17.0.2-1733450198398 heartbeating to localhost/127.0.0.1:43593 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-450157059-172.17.0.2-1733450198398 (Datanode Uuid c846784d-567f-4498-9227-66696fb12e44) service to localhost/127.0.0.1:43593
2024-12-06T01:56:41,077 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data1/current/BP-450157059-172.17.0.2-1733450198398 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:41,077 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/cluster_97890684-f68b-c130-4298-f1f56b0e93bf/data/data2/current/BP-450157059-172.17.0.2-1733450198398 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-06T01:56:41,077 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-06T01:56:41,078 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-06T01:56:41,078 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-06T01:56:41,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65b573aa{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-06T01:56:41,084 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@50345f99{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-06T01:56:41,084 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-06T01:56:41,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@322729b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-06T01:56:41,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15cc83a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f895c614-2311-731d-2a65-1f697f071225/hadoop.log.dir/,STOPPED}
2024-12-06T01:56:41,093 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-06T01:56:41,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-06T01:56:41,125 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 228)
Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43593
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43593
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43593
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:43593 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43593
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:43593 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (1609174458) connection to localhost/127.0.0.1:43593 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:43593
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=535 (was 506) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=8912 (was 8928)